OLD | NEW |
1 # -*- coding: utf-8 -*- | 1 # -*- coding: utf-8 -*- |
2 """The task multi-process processing engine.""" | 2 """The task multi-process processing engine.""" |
3 | 3 |
4 from __future__ import unicode_literals | 4 from __future__ import unicode_literals |
5 | 5 |
6 import heapq | 6 import heapq |
7 import logging | 7 import logging |
8 import multiprocessing | 8 import multiprocessing |
9 import os | 9 import os |
10 import time | 10 import time |
(...skipping 229 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
240 fully_merged = True | 240 fully_merged = True |
241 | 241 |
242 if self._processing_profiler: | 242 if self._processing_profiler: |
243 self._processing_profiler.StopTiming('merge') | 243 self._processing_profiler.StopTiming('merge') |
244 | 244 |
245 if fully_merged: | 245 if fully_merged: |
246 try: | 246 try: |
247 self._task_manager.CompleteTask(self._merge_task) | 247 self._task_manager.CompleteTask(self._merge_task) |
248 except KeyError as exception: | 248 except KeyError as exception: |
249 logger.error( | 249 logger.error( |
250 'Unable to complete task {0:s}, with Error {1:s}'.format( | 250 'Unable to complete task: {0:s}, with error: {1!s}'.format( |
251 self._merge_task.identifier, exception)) | 251 self._merge_task.identifier, exception)) |
252 | 252 |
253 if self._storage_merge_reader_on_hold: | 253 if self._storage_merge_reader_on_hold: |
254 self._merge_task = self._merge_task_on_hold | 254 self._merge_task = self._merge_task_on_hold |
255 self._storage_merge_reader = self._storage_merge_reader_on_hold | 255 self._storage_merge_reader = self._storage_merge_reader_on_hold |
256 | 256 |
257 self._merge_task_on_hold = None | 257 self._merge_task_on_hold = None |
258 self._storage_merge_reader_on_hold = None | 258 self._storage_merge_reader_on_hold = None |
259 else: | 259 else: |
260 self._merge_task = None | 260 self._merge_task = None |
(...skipping 584 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
845 self._worker_memory_limit = definitions.DEFAULT_WORKER_MEMORY_LIMIT | 845 self._worker_memory_limit = definitions.DEFAULT_WORKER_MEMORY_LIMIT |
846 | 846 |
847 self._processing_configuration = None | 847 self._processing_configuration = None |
848 | 848 |
849 self._filter_find_specs = None | 849 self._filter_find_specs = None |
850 self._session_identifier = None | 850 self._session_identifier = None |
851 self._status_update_callback = None | 851 self._status_update_callback = None |
852 self._storage_writer = None | 852 self._storage_writer = None |
853 | 853 |
854 return self._processing_status | 854 return self._processing_status |
OLD | NEW |