projects
/
osm
/
SO.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Updated changes from IM module to introduce operational state upgrading
[osm/SO.git]
/
common
/
python
/
rift
/
downloader
/
url.py
diff --git
a/common/python/rift/downloader/url.py
b/common/python/rift/downloader/url.py
index
2768894
..
3aa9ac2
100644
(file)
--- a/
common/python/rift/downloader/url.py
+++ b/
common/python/rift/downloader/url.py
@@
-23,7
+23,6
@@
import os
import tempfile
import threading
import time
import tempfile
import threading
import time
-import uuid
import zlib
import requests
import zlib
import requests
@@
-34,12
+33,8
@@
from requests.packages.urllib3.util.retry import Retry
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
-import gi
-gi.require_version("RwPkgMgmtYang", "1.0")
-
-from gi.repository import RwPkgMgmtYang
from . import base
from . import base
-
+from .local_file import LocalFileAdapter as LocalFileAdapter
class UrlDownloader(base.AbstractDownloader):
"""Handles downloads of URL with some basic retry strategy.
class UrlDownloader(base.AbstractDownloader):
"""Handles downloads of URL with some basic retry strategy.
@@
-106,9
+101,12
@@
class UrlDownloader(base.AbstractDownloader):
def _create_session(self):
session = requests.Session()
def _create_session(self):
session = requests.Session()
- retries = Retry(total=5, backoff_factor=1)
+ # 3 connection attempts should be more than enough, We can't wait forever!
+ # The user needs to be updated of the status
+ retries = Retry(total=2, backoff_factor=1)
session.mount("http://", HTTPAdapter(max_retries=retries))
session.mount("https://", HTTPAdapter(max_retries=retries))
session.mount("http://", HTTPAdapter(max_retries=retries))
session.mount("https://", HTTPAdapter(max_retries=retries))
+ session.mount("file://", LocalFileAdapter())
return session
return session
@@
-153,8
+151,8
@@
class UrlDownloader(base.AbstractDownloader):
try:
os.remove(self.filepath)
try:
os.remove(self.filepath)
- except Exception
as e
:
- self.log.exception(e)
+ except Exception:
+ pass
def download(self):
"""Start the download
def download(self):
"""Start the download
@@
-186,18
+184,21
@@
class UrlDownloader(base.AbstractDownloader):
def _download(self):
def _download(self):
- url_options = {"verify": False}
+ url_options = {"verify": False
, "timeout": 10
}
if self.auth is not None:
url_options["auth"] = self.auth
response = self.session.head(self.url, **url_options)
if self.auth is not None:
url_options["auth"] = self.auth
response = self.session.head(self.url, **url_options)
+ if response.status_code != requests.codes.ok:
+ response.raise_for_status()
+
# Prepare the meta data
self.meta.update_data_with_head(response.headers)
self.meta.start_download()
# Prepare the meta data
self.meta.update_data_with_head(response.headers)
self.meta.start_download()
- self.download_
started
()
+ self.download_
progress
()
url_options["stream"] = True,
request = self.session.get(self.url, **url_options)
url_options["stream"] = True,
request = self.session.get(self.url, **url_options)
@@
-219,7
+220,7
@@
class UrlDownloader(base.AbstractDownloader):
chunk = self.check_and_decompress(chunk)
self._fh.write(chunk)
chunk = self.check_and_decompress(chunk)
self._fh.write(chunk)
- self.download_progress()
+
#
self.download_progress()
self.meta.end_download()
self.close()
self.meta.end_download()
self.close()