update from RIFT as of 696b75d2fe9fb046261b08c616f1bcf6c0b54a9b second try

[osm/SO.git] / common / python / rift / downloader / url.py

diff --git a/common/python/rift/downloader/url.py b/common/python/rift/downloader/url.py
index 2768894..3aa9ac2 100644 (file)
--- a/common/python/rift/downloader/url.py
+++ b/common/python/rift/downloader/url.py
@@ -23,7 +23,6 @@ import os
 import tempfile
 import threading
 import time
-import uuid
 import zlib
 
 import requests
@@ -34,12 +33,8 @@ from requests.packages.urllib3.util.retry import Retry
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
-import gi
-gi.require_version("RwPkgMgmtYang", "1.0")
-
-from gi.repository import RwPkgMgmtYang
 from . import base
-
+from .local_file import LocalFileAdapter as LocalFileAdapter
 
 class UrlDownloader(base.AbstractDownloader):
     """Handles downloads of URL with some basic retry strategy.
@@ -106,9 +101,12 @@ class UrlDownloader(base.AbstractDownloader):
 
     def _create_session(self):
         session = requests.Session()
-        retries = Retry(total=5, backoff_factor=1)
+        # 3 connection attempts should be more than enough, We can't wait forever!
+        # The user needs to be  updated of the status
+        retries = Retry(total=2, backoff_factor=1)
         session.mount("http://", HTTPAdapter(max_retries=retries))
         session.mount("https://", HTTPAdapter(max_retries=retries))
+        session.mount("file://", LocalFileAdapter())
 
         return session
 
@@ -153,8 +151,8 @@ class UrlDownloader(base.AbstractDownloader):
 
             try:
                 os.remove(self.filepath)
-            except Exception as e:
-                self.log.exception(e)
+            except Exception:
+                pass
 
     def download(self):
         """Start the download
@@ -186,18 +184,21 @@ class UrlDownloader(base.AbstractDownloader):
 
     def _download(self):
 
-        url_options = {"verify": False}
+        url_options = {"verify": False, "timeout": 10}
 
         if self.auth is not None:
             url_options["auth"] = self.auth
 
         response = self.session.head(self.url, **url_options)
 
+        if response.status_code != requests.codes.ok:
+            response.raise_for_status()
+
         # Prepare the meta data
         self.meta.update_data_with_head(response.headers)
         self.meta.start_download()
 
-        self.download_started()
+        self.download_progress()
 
         url_options["stream"] = True,
         request = self.session.get(self.url, **url_options)
@@ -219,7 +220,7 @@ class UrlDownloader(base.AbstractDownloader):
                 chunk = self.check_and_decompress(chunk)
 
                 self._fh.write(chunk)
-                self.download_progress()
+                #self.download_progress()
 
         self.meta.end_download()
         self.close()