2 # Copyright 2016 RIFT.IO Inc
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # Author(s): Varun Prasad
17 # Creation Date: 09/25/2016
30 import requests
.exceptions
31 from requests
.adapters
import HTTPAdapter
32 from requests
.packages
.urllib3
.util
.retry
import Retry
33 # disable unsigned certificate warning
34 from requests
.packages
.urllib3
.exceptions
import InsecureRequestWarning
35 requests
.packages
.urllib3
.disable_warnings(InsecureRequestWarning
)
38 gi
.require_version("RwPkgMgmtYang", "1.0")
40 from gi
.repository
import RwPkgMgmtYang
42 from .local_file
import LocalFileAdapter
as LocalFileAdapter
45 class UrlDownloader(base
.AbstractDownloader
):
46 """Handles downloads of URL with some basic retry strategy.
53 decompress_on_fly
=False,
57 model (str or DownloadJob): Url string to download or the Yang model
58 file_obj (str,file handle): Optional, If not set we create a temp
59 location to store the file.
60 delete_on_fail (bool, optional): Clean up the partially downloaded
61 file, if the download failed or was canceled
62 callback_handler (None, optional): Instance of base.DownloaderCallbackHandler
66 self
.log
= log
or logging
.getLogger()
67 self
.log
.setLevel(logging
.DEBUG
)
69 self
._fh
, filename
= self
._validate
_fn
(file_obj
)
70 self
.meta
= base
.DownloadMeta(url
, filename
)
72 self
.session
= self
._create
_session
()
73 self
._cancel
_event
= threading
.Event()
76 self
.delete_on_fail
= delete_on_fail
78 self
.decompress_on_fly
= decompress_on_fly
79 self
._decompressor
= zlib
.decompressobj(16 + zlib
.MAX_WBITS
)
82 data
= {"model": self
.meta
.as_dict()}
85 def _validate_fn(self
, file_obj
):
87 If no file object is given then create a temp file
88 if a filename is given open the file in wb mode
90 Finally verify the mode open mode of the file
94 _
, file_obj
= tempfile
.mkstemp()
96 file_obj
= open(file_obj
, "wb")
98 # If the fh is a filename
99 if type(file_obj
) is str:
100 file_obj
= open(file_obj
, "wb")
102 if type(file_obj
) is not io
.BufferedWriter
:
103 raise base
.InvalidDestinationError("Destination file cannot be"
106 return file_obj
, file_obj
.name
108 def _create_session(self
):
109 session
= requests
.Session()
110 retries
= Retry(total
=5, backoff_factor
=1)
111 session
.mount("http://", HTTPAdapter(max_retries
=retries
))
112 session
.mount("https://", HTTPAdapter(max_retries
=retries
))
113 session
.mount("file://", LocalFileAdapter())
117 def update_data_from_headers(self
, headers
):
118 """Update the model from the header of HEAD request
121 headers (dict): headers from HEAD response
123 self
.meta
.bytes_total
= 0
124 if 'Content-Length' in headers
:
125 self
.meta
.bytes_total
= int(headers
['Content-Length'])
126 self
.meta
.progress_percent
= 0
127 self
.meta
.bytes_downloaded
= 0
135 return self
.meta
.filepath
137 # Start of override methods
139 def download_id(self
):
140 return self
.meta
.download_id
142 def cancel_download(self
):
143 self
._cancel
_event
.set()
150 """Remove the file if the download failed.
152 if self
.meta
.status
in [base
.DownloadStatus
.FAILED
, base
.DownloadStatus
.CANCELLED
] and self
.delete_on_fail
:
153 self
.log
.info("Cleaning up failed download and removing {}".format(
157 os
.remove(self
.filepath
)
158 except Exception as e
:
159 self
.log
.exception(e
)
162 """Start the download
164 Trigger an HEAD request to get the meta data before starting the download
168 except Exception as e
:
169 self
.log
.exception(str(e
))
170 self
.meta
.detail
= str(e
)
171 self
.meta
.stop_time
= time
.time()
173 self
.download_failed()
175 # Close all file handles and clean up
179 self
.download_finished()
181 # end of override methods
183 def check_and_decompress(self
, chunk
):
184 if self
.url
.endswith(".gz") and self
.decompress_on_fly
:
185 chunk
= self
._decompressor
.decompress(chunk
)
191 url_options
= {"verify": False}
193 if self
.auth
is not None:
194 url_options
["auth"] = self
.auth
196 response
= self
.session
.head(self
.url
, **url_options
)
198 # Prepare the meta data
199 self
.meta
.update_data_with_head(response
.headers
)
200 self
.meta
.start_download()
202 self
.download_started()
204 url_options
["stream"] = True,
205 request
= self
.session
.get(self
.url
, **url_options
)
207 if request
.status_code
!= requests
.codes
.ok
:
208 request
.raise_for_status()
210 # actual start time, excluding the HEAD request.
211 for chunk
in request
.iter_content(chunk_size
=1024 * 50):
212 if self
._cancel
_event
.is_set():
213 self
.log
.info("Download of URL {} to {} has been cancelled".format(
214 self
.url
, self
.filepath
))
217 if chunk
: # filter out keep-alive new chunks
218 self
.meta
.update_with_data(chunk
)
219 self
.log
.debug("Download progress: {}".format(self
.meta
.as_dict()))
221 chunk
= self
.check_and_decompress(chunk
)
223 self
._fh
.write(chunk
)
224 self
.download_progress()
226 self
.meta
.end_download()
229 if self
._cancel
_event
.is_set():
230 self
.download_cancelled()
232 self
.download_succeeded()
236 # Start of delegate calls
237 def call_delegate(self
, event
):
238 if not self
.delegate
:
241 getattr(self
.delegate
, event
)(self
.meta
)
243 def download_failed(self
):
244 self
.meta
.set_state(base
.DownloadStatus
.FAILED
)
245 self
.call_delegate("on_download_failed")
247 def download_cancelled(self
):
248 self
.meta
.detail
= "Download canceled by user."
249 self
.meta
.set_state(base
.DownloadStatus
.CANCELLED
)
250 self
.call_delegate("on_download_cancelled")
252 def download_progress(self
):
253 self
.meta
.detail
= "Download in progress."
254 self
.meta
.set_state(base
.DownloadStatus
.IN_PROGRESS
)
255 self
.call_delegate("on_download_progress")
257 def download_succeeded(self
):
258 self
.meta
.detail
= "Download completed successfully."
259 self
.meta
.set_state(base
.DownloadStatus
.COMPLETED
)
260 self
.call_delegate("on_download_succeeded")
262 def download_started(self
):
263 self
.meta
.detail
= "Setting up download and extracting meta."
264 self
.meta
.set_state(base
.DownloadStatus
.STARTED
)
265 self
.call_delegate("on_download_started")
267 def download_finished(self
):
268 self
.call_delegate("on_download_finished")