LEFT | RIGHT |
(no file at all) | |
1 # -*- coding: utf-8 -*- | 1 # -*- coding: utf-8 -*- |
2 # Copyright 2013 Google Inc. All Rights Reserved. | 2 # Copyright 2013 Google Inc. All Rights Reserved. |
3 # | 3 # |
4 # Licensed under the Apache License, Version 2.0 (the "License"); | 4 # Licensed under the Apache License, Version 2.0 (the "License"); |
5 # you may not use this file except in compliance with the License. | 5 # you may not use this file except in compliance with the License. |
6 # You may obtain a copy of the License at | 6 # You may obtain a copy of the License at |
7 # | 7 # |
8 # http://www.apache.org/licenses/LICENSE-2.0 | 8 # http://www.apache.org/licenses/LICENSE-2.0 |
9 # | 9 # |
10 # Unless required by applicable law or agreed to in writing, software | 10 # Unless required by applicable law or agreed to in writing, software |
11 # distributed under the License is distributed on an "AS IS" BASIS, | 11 # distributed under the License is distributed on an "AS IS" BASIS, |
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 # See the License for the specific language governing permissions and | 13 # See the License for the specific language governing permissions and |
14 # limitations under the License. | 14 # limitations under the License. |
15 """Utility classes and methods for the parallelism framework.""" | 15 """Utility classes and methods for the parallelism framework.""" |
16 | 16 |
17 from __future__ import absolute_import | 17 from __future__ import absolute_import |
18 from __future__ import print_function | 18 from __future__ import print_function |
19 | 19 |
20 import collections | 20 import collections |
21 import errno | 21 import errno |
| 22 import logging |
22 import multiprocessing | 23 import multiprocessing |
23 import threading | 24 import threading |
24 import traceback | 25 import traceback |
25 | 26 |
26 from gslib.utils import constants | 27 from gslib.utils import constants |
27 from gslib.utils import system_util | 28 from gslib.utils import system_util |
28 from six.moves import queue as Queue | 29 from six.moves import queue as Queue |
29 | 30 |
30 # pylint: disable=g-import-not-at-top | 31 # pylint: disable=g-import-not-at-top |
31 try: | 32 try: |
(...skipping 212 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
244 (bool indicator if multiprocessing should be prohibited, OS name) | 245 (bool indicator if multiprocessing should be prohibited, OS name) |
245 """ | 246 """ |
246 if system_util.IS_WINDOWS: | 247 if system_util.IS_WINDOWS: |
247 return (True, 'Windows') | 248 return (True, 'Windows') |
248 try: | 249 try: |
249 with open('/etc/os-release', 'r') as f: | 250 with open('/etc/os-release', 'r') as f: |
250 os_name = f.read().split('\n')[0].split('=')[1].strip('"') | 251 os_name = f.read().split('\n')[0].split('=')[1].strip('"') |
251 return ('alpine linux' in os_name.lower(), os_name) | 252 return ('alpine linux' in os_name.lower(), os_name) |
252 except IOError as e: | 253 except IOError as e: |
253 if e.errno == errno.ENOENT: | 254 if e.errno == errno.ENOENT: |
254 logger.debug('Unable to open /etc/os-release to determine whether OS ' | 255 logging.debug('Unable to open /etc/os-release to determine whether OS ' |
255 'supports multiprocessing: errno=%d, message=%s' | 256 'supports multiprocessing: errno=%d, message=%s' |
256 % (e.errno, e.message)) | 257 % (e.errno, e.message)) |
257 return (False, 'Unknown') | 258 return (False, 'Unknown') |
258 else: | 259 else: |
259 raise | 260 raise |
260 | 261 |
261 | 262 |
262 def CheckMultiprocessingAvailableAndInit(logger=None): | 263 def CheckMultiprocessingAvailableAndInit(logger=None): |
263 """Checks if multiprocessing is available, and if so performs initialization. | 264 """Checks if multiprocessing is available, and if so performs initialization. |
264 | 265 |
265 There are some environments in which there is no way to use multiprocessing | 266 There are some environments in which there is no way to use multiprocessing |
266 logic that's built into Python (e.g., if /dev/shm is not available, then | 267 logic that's built into Python (e.g., if /dev/shm is not available, then |
(...skipping 150 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
417 timeout: (optional) amount of time to wait before repeating put request. | 418 timeout: (optional) amount of time to wait before repeating put request. |
418 """ | 419 """ |
419 put_success = False | 420 put_success = False |
420 while not put_success: | 421 while not put_success: |
421 try: | 422 try: |
422 queue.put(msg, timeout=timeout) | 423 queue.put(msg, timeout=timeout) |
423 put_success = True | 424 put_success = True |
424 except Queue.Full: | 425 except Queue.Full: |
425 pass | 426 pass |
426 # pylint: enable=invalid-name | 427 # pylint: enable=invalid-name |
LEFT | RIGHT |