Left: | ||
Right: |
LEFT | RIGHT |
---|---|
1 # -*- coding: utf-8 -*- | 1 # -*- coding: utf-8 -*- |
2 """SQLite-based storage.""" | 2 """SQLite-based storage.""" |
3 | 3 |
4 import os | 4 import os |
5 import sqlite3 | 5 import sqlite3 |
6 | 6 |
7 from plaso.lib import definitions | 7 from plaso.lib import definitions |
8 from plaso.storage import identifiers | 8 from plaso.storage import identifiers |
9 from plaso.storage import interface | 9 from plaso.storage import interface |
10 | 10 |
11 | 11 |
12 class SQLiteStorageFile(interface.BaseFileStorage): | 12 class SQLiteStorageFile(interface.BaseFileStorage): |
13 """SQLite-based storage file. | 13 """SQLite-based storage file. |
14 | 14 |
15 Attributes: | 15 Attributes: |
16 format_version (int): storage format version. | 16 format_version (int): storage format version. |
17 serialization_format (str): serialization format. | 17 serialization_format (str): serialization format. |
18 storage_type (str): storage type. | 18 storage_type (str): storage type. |
19 """ | 19 """ |
20 | 20 |
21 _FORMAT_VERSION = 20170121 | 21 _FORMAT_VERSION = 20170121 |
22 | 22 |
23 _CONTAINER_TYPES = ( | 23 _CONTAINER_TYPES = ( |
24 u'analysis_report', u'extraction_error', u'event', u'event_source', | 24 u'analysis_report', u'extraction_error', u'event', u'event_source', |
25 u'event_tag', u'session_completion', u'session_start', | 25 u'event_tag', u'session_completion', u'session_start', |
26 u'task_completion', u'task_start') | 26 u'task_completion', u'task_start') |
27 | 27 |
28 _CREATE_TABLE_QUERY = ( | 28 _CREATE_TABLE_QUERY = ( |
29 u'CREATE TABLE {0:s} (' | 29 u'CREATE TABLE {0:s} (' |
onager
2017/03/29 12:52:02
Please don't do this, use prepared statements inst
Joachim Metz
2017/04/08 18:11:31
The code fully controls {0:s} and is not based on
| |
30 u'_identifier INTEGER PRIMARY KEY AUTOINCREMENT,' | 30 u'_identifier INTEGER PRIMARY KEY AUTOINCREMENT,' |
31 u'_data ATTRIBUTE_CONTAINER);') | 31 u'_data ATTRIBUTE_CONTAINER);') |
32 | 32 |
33 _HAS_TABLE_QUERY = ( | 33 _HAS_TABLE_QUERY = ( |
34 u'SELECT name FROM sqlite_master ' | 34 u'SELECT name FROM sqlite_master ' |
35 u'WHERE type = "table" AND name = "{0:s}"') | 35 u'WHERE type = "table" AND name = "{0:s}"') |
36 | 36 |
37 def __init__(self, storage_type=definitions.STORAGE_TYPE_SESSION): | 37 def __init__(self, storage_type=definitions.STORAGE_TYPE_SESSION): |
38 """Initializes a storage. | 38 """Initializes a storage. |
39 | 39 |
(...skipping 14 matching lines...) Expand all Loading... | |
54 | 54 |
55 def _GetAttributeContainer(self, container_type): | 55 def _GetAttributeContainer(self, container_type): |
56 """Retrieves an attribute container. | 56 """Retrieves an attribute container. |
57 | 57 |
58 Args: | 58 Args: |
59 container_type (str): attribute container type. | 59 container_type (str): attribute container type. |
60 | 60 |
61 Yields: | 61 Yields: |
62 AttributeContainer: attribute container. | 62 AttributeContainer: attribute container. |
63 """ | 63 """ |
64 query = u'SELECT _identifier, _data FROM {0:s}'.format(container_type) | 64 query = u'SELECT _identifier, _data FROM {0:s}'.format(container_type) |
onager
2017/03/29 12:52:03
See above re: prepared statements.
Joachim Metz
2017/04/08 18:11:31
The code fully controls {0:s} and is not based on
| |
65 self._cursor.execute(query) | 65 self._cursor.execute(query) |
66 | 66 |
67 row = self._cursor.fetchone() | 67 row = self._cursor.fetchone() |
68 while row: | 68 while row: |
69 identifier = identifiers.SQLTableIdentifier(container_type, row[0]) | 69 identifier = identifiers.SQLTableIdentifier(container_type, row[0]) |
70 | 70 |
71 attribute_container = row[1] | 71 attribute_container = row[1] |
72 attribute_container.SetIdentifier(identifier) | 72 attribute_container.SetIdentifier(identifier) |
73 yield attribute_container | 73 yield attribute_container |
74 | 74 |
75 row = self._cursor.fetchone() | 75 row = self._cursor.fetchone() |
76 | 76 |
77 def _HasTable(self, table_name): | 77 def _HasTable(self, table_name): |
78 """Determines if a specific table exists. | 78 """Determines if a specific table exists. |
79 | 79 |
80 Args: | 80 Args: |
81 table_name: the table name. | 81 table_name: the table name. |
82 | 82 |
83 Returns: | 83 Returns: |
84 True if the table exists, false otheriwse. | 84 True if the table exists, false otheriwse. |
85 """ | 85 """ |
86 query = self._HAS_TABLE_QUERY.format(table_name) | 86 query = self._HAS_TABLE_QUERY.format(table_name) |
87 | 87 |
88 self._cursor.execute(query) | 88 self._cursor.execute(query) |
89 return bool(self._cursor.fetchone()) | 89 return bool(self._cursor.fetchone()) |
90 | 90 |
91 def _WriteAttributeContainer(self, attribute_container): | 91 def _WriteAttributeContainer(self, attribute_container): |
92 """Writes an attribute container. | 92 """Writes an attribute container. |
onager
2017/03/29 12:52:02
+note that the table for the container type must e
Joachim Metz
2017/04/08 18:11:31
Done.
| |
93 | 93 |
94 The table for the container type must exist. | |
95 | |
94 Args: | 96 Args: |
95 attribute_container (AttributeContainer): attribute container. | 97 attribute_container (AttributeContainer): attribute container. |
96 | 98 |
97 Raises: | 99 Raises: |
98 IOError: when the storage file is closed or read-only. | 100 IOError: when the storage file is closed or read-only. |
99 """ | 101 """ |
100 if not self._is_open: | 102 if not self._is_open: |
101 raise IOError(u'Unable to write to closed storage file.') | 103 raise IOError(u'Unable to write to closed storage file.') |
102 | 104 |
103 if self._read_only: | 105 if self._read_only: |
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
214 return self._GetAttributeContainer(u'event') | 216 return self._GetAttributeContainer(u'event') |
215 | 217 |
216 def GetEventSourceByIndex(self, index): | 218 def GetEventSourceByIndex(self, index): |
217 """Retrieves a specific event source. | 219 """Retrieves a specific event source. |
218 | 220 |
219 Args: | 221 Args: |
220 index (int): event source index. | 222 index (int): event source index. |
221 | 223 |
222 Returns: | 224 Returns: |
223 EventSource: event source or None. | 225 EventSource: event source or None. |
224 | |
225 Raises: | |
onager
2017/03/29 12:52:03
This doesn't raise, remove.
Joachim Metz
2017/04/08 18:11:31
Done.
| |
226 IOError: if a stream is missing. | |
227 """ | 226 """ |
228 query = u'SELECT _data FROM {0:s} WHERE rowid = {1:d}'.format( | 227 query = u'SELECT _data FROM {0:s} WHERE rowid = {1:d}'.format( |
229 u'event_source', index + 1) | 228 u'event_source', index + 1) |
230 self._cursor.execute(query) | 229 self._cursor.execute(query) |
231 | 230 |
232 row = self._cursor.fetchone() | 231 row = self._cursor.fetchone() |
233 if row: | 232 if row: |
234 return row[0] | 233 return row[0] |
235 | 234 |
236 def GetEventSources(self): | 235 def GetEventSources(self): |
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
302 query = u'SELECT COUNT(*) FROM {0:s}'.format(u'event_tags') | 301 query = u'SELECT COUNT(*) FROM {0:s}'.format(u'event_tags') |
303 self._cursor.execute(query) | 302 self._cursor.execute(query) |
304 | 303 |
305 row = self._cursor.fetchone() | 304 row = self._cursor.fetchone() |
306 return row and row[0] != 0 | 305 return row and row[0] != 0 |
307 | 306 |
308 def Open(self, path=None, read_only=True, **unused_kwargs): | 307 def Open(self, path=None, read_only=True, **unused_kwargs): |
309 """Opens the storage. | 308 """Opens the storage. |
310 | 309 |
311 Args: | 310 Args: |
312 path (Optional[str]): path of the storage file. | 311 path (Optional[str]): path to the storage file. |
onager
2017/03/29 12:52:02
-of +to
Joachim Metz
2017/04/08 18:11:31
Done.
| |
313 read_only (Optional[bool]): True if the file should be opened in | 312 read_only (Optional[bool]): True if the file should be opened in |
314 read-only mode. | 313 read-only mode. |
315 | 314 |
316 Raises: | 315 Raises: |
317 IOError: if the storage file is already opened. | 316 IOError: if the storage file is already opened or if the database |
317 cannot be connected. | |
318 ValueError: if path is missing. | 318 ValueError: if path is missing. |
319 """ | 319 """ |
320 if self._is_open: | 320 if self._is_open: |
321 raise IOError(u'Storage file already opened.') | 321 raise IOError(u'Storage file already opened.') |
322 | 322 |
323 if not path: | 323 if not path: |
324 raise ValueError(u'Missing path.') | 324 raise ValueError(u'Missing path.') |
325 | 325 |
326 self._connection = sqlite3.connect( | 326 connection = sqlite3.connect( |
327 path, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) | 327 path, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) |
328 if not self._connection: | 328 |
329 cursor = connection.cursor() | |
330 if not cursor: | |
329 return False | 331 return False |
onager
2017/03/29 12:52:03
Why doesn't this raise? Can connect() really retur
Joachim Metz
2017/04/08 18:11:31
Documenation is not clear about this:
https://docs
| |
330 | 332 |
331 self._cursor = self._connection.cursor() | 333 self._connection = connection |
332 if not self._cursor: | 334 self._cursor = cursor |
333 return False | 335 self._is_open = True |
336 self._read_only = read_only | |
334 | 337 |
335 if not read_only: | 338 if not read_only: |
336 for container_type in self._CONTAINER_TYPES: | 339 for container_type in self._CONTAINER_TYPES: |
337 if not self._HasTable(container_type): | 340 if not self._HasTable(container_type): |
338 query = self._CREATE_TABLE_QUERY.format(container_type) | 341 query = self._CREATE_TABLE_QUERY.format(container_type) |
339 self._cursor.execute(query) | 342 self._cursor.execute(query) |
340 | 343 |
341 self._connection.commit() | 344 self._connection.commit() |
342 | 345 |
343 self._is_open = True | |
344 self._read_only = read_only | |
345 | |
346 def ReadPreprocessingInformation(self, knowledge_base): | 346 def ReadPreprocessingInformation(self, knowledge_base): |
347 """Reads preprocessing information. | 347 """Reads preprocessing information. |
348 | 348 |
349 The preprocessing information contains the system configuration which | 349 The preprocessing information contains the system configuration which |
350 contains information about various system specific configuration data, | 350 contains information about various system specific configuration data, |
351 for example the user accounts. | 351 for example the user accounts. |
352 | 352 |
353 Args: | 353 Args: |
354 knowledge_base (KnowledgeBase): is used to store the preprocessing | 354 knowledge_base (KnowledgeBase): is used to store the preprocessing |
355 information. | 355 information. |
356 """ | 356 """ |
357 # TODO: implement. | |
357 pass | 358 pass |
onager
2017/03/29 12:52:02
+TODO...
Joachim Metz
2017/04/08 18:11:31
Done.
| |
358 | 359 |
359 def WritePreprocessingInformation(self, knowledge_base): | 360 def WritePreprocessingInformation(self, knowledge_base): |
360 """Writes preprocessing information. | 361 """Writes preprocessing information. |
361 | 362 |
362 Args: | 363 Args: |
363 knowledge_base (KnowledgeBase): contains the preprocessing information. | 364 knowledge_base (KnowledgeBase): contains the preprocessing information. |
364 | 365 |
365 Raises: | 366 Raises: |
366 IOError: if the storage type does not support writing preprocess | 367 IOError: if the storage type does not support writing preprocess |
367 information or the storage file is closed or read-only or | 368 information or the storage file is closed or read-only or |
368 if the preprocess information stream already exists. | 369 if the preprocess information stream already exists. |
369 """ | 370 """ |
371 # TODO: implement. | |
370 pass | 372 pass |
onager
2017/03/29 12:52:02
+TODO...
Joachim Metz
2017/04/08 18:11:31
Done.
| |
371 | 373 |
372 def WriteSessionCompletion(self, session_completion): | 374 def WriteSessionCompletion(self, session_completion): |
373 """Writes session completion information. | 375 """Writes session completion information. |
374 | 376 |
375 Args: | 377 Args: |
376 session_completion (SessionCompletion): session completion information. | 378 session_completion (SessionCompletion): session completion information. |
377 """ | 379 """ |
378 self._WriteAttributeContainer(session_completion) | 380 self._WriteAttributeContainer(session_completion) |
379 | 381 |
380 def WriteSessionStart(self, session_start): | 382 def WriteSessionStart(self, session_start): |
(...skipping 15 matching lines...) Expand all Loading... | |
396 def WriteTaskStart(self, task_start): | 398 def WriteTaskStart(self, task_start): |
397 """Writes task start information. | 399 """Writes task start information. |
398 | 400 |
399 Args: | 401 Args: |
400 task_start (TaskStart): task start information. | 402 task_start (TaskStart): task start information. |
401 """ | 403 """ |
402 self._WriteAttributeContainer(task_start) | 404 self._WriteAttributeContainer(task_start) |
403 | 405 |
404 | 406 |
405 class SQLiteStorageMergeReader(interface.StorageMergeReader): | 407 class SQLiteStorageMergeReader(interface.StorageMergeReader): |
406 """Class that implements a SQLite-based storage file reader for merging.""" | 408 """SQLite-based storage file reader for merging.""" |
407 | 409 |
408 _CONTAINER_TYPES = ( | 410 _CONTAINER_TYPES = ( |
409 u'event_source', u'event', u'extraction_error', u'analysis_report', | 411 u'event_source', u'event', u'extraction_error', u'analysis_report', |
410 u'event_tag') | 412 u'event_tag') |
411 | 413 |
412 def __init__(self, storage_writer, path): | 414 def __init__(self, storage_writer, path): |
413 """Initializes a storage merge reader. | 415 """Initializes a storage merge reader. |
414 | 416 |
415 Args: | 417 Args: |
416 storage_writer (StorageWriter): storage writer. | 418 storage_writer (StorageWriter): storage writer. |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
463 maximum_number_of_containers (Optional[int]): maximum number of | 465 maximum_number_of_containers (Optional[int]): maximum number of |
464 containers to merge, where 0 represent no limit. | 466 containers to merge, where 0 represent no limit. |
465 | 467 |
466 Returns: | 468 Returns: |
467 bool: True if the entire task storage file has been merged. | 469 bool: True if the entire task storage file has been merged. |
468 | 470 |
469 Raises: | 471 Raises: |
470 OSError: if the task storage file cannot be deleted. | 472 OSError: if the task storage file cannot be deleted. |
471 """ | 473 """ |
472 # TODO: add support for maximum_number_of_containers. | 474 # TODO: add support for maximum_number_of_containers. |
473 connection = sqlite3.connect( | 475 connection = sqlite3.connect( |
onager
2017/03/29 12:52:03
You aren't checking for connection is None here, b
Joachim Metz
2017/04/08 18:11:31
Documenation is not clear about this:
https://docs
| |
474 self._path, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) | 476 self._path, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) |
475 cursor = connection.cursor() | 477 cursor = connection.cursor() |
476 | 478 |
477 number_of_containers = 0 | 479 number_of_containers = 0 |
478 for container_type in self._CONTAINER_TYPES: | 480 for container_type in self._CONTAINER_TYPES: |
onager
2017/03/29 12:52:02
What if not all the container types tables are in
Joachim Metz
2017/04/08 18:11:31
The code at line 335 should ensure the tables exis
| |
481 if not self._HasTable(container_type): | |
482 continue | |
483 | |
479 query = u'SELECT _data FROM {0:s}'.format(container_type) | 484 query = u'SELECT _data FROM {0:s}'.format(container_type) |
480 cursor.execute(query) | 485 cursor.execute(query) |
481 | 486 |
482 row = cursor.fetchone() | 487 row = cursor.fetchone() |
483 while row: | 488 while row: |
484 self._AddAttributeContainer(row[0]) | 489 self._AddAttributeContainer(row[0]) |
485 number_of_containers += 1 | 490 number_of_containers += 1 |
486 | 491 |
487 # if (maximum_number_of_containers > 0 and | 492 # if (maximum_number_of_containers > 0 and |
488 # number_of_containers >= maximum_number_of_containers): | 493 # number_of_containers >= maximum_number_of_containers): |
489 # return False | 494 # return False |
490 | 495 |
491 row = cursor.fetchone() | 496 row = cursor.fetchone() |
492 | 497 |
493 # pylint: disable=protected-access | 498 # pylint: disable=protected-access |
onager
2017/03/29 12:52:02
Please add a todo to fix this hack.
Joachim Metz
2017/04/08 18:11:31
Acknowledged.
| |
499 # TODO: fix this hack. | |
494 self._storage_writer._storage_file._connection.commit() | 500 self._storage_writer._storage_file._connection.commit() |
495 | 501 |
496 connection.close() | 502 connection.close() |
497 | 503 |
498 connection = None | 504 connection = None |
499 cursor = None | 505 cursor = None |
500 | 506 |
501 os.remove(self._path) | 507 os.remove(self._path) |
502 | 508 |
503 return True | 509 return True |
504 | 510 |
505 | 511 |
506 class SQLiteStorageFileWriter(interface.FileStorageWriter): | 512 class SQLiteStorageFileWriter(interface.FileStorageWriter): |
507 """Class that implements the SQLite-based storage file writer.""" | 513 """SQLite-based storage file writer.""" |
508 | 514 |
509 def _CreateStorageFile(self): | 515 def _CreateStorageFile(self): |
510 """Creates a storage file. | 516 """Creates a storage file. |
511 | 517 |
512 Returns: | 518 Returns: |
513 BaseFileStorage: storage file. | 519 SQLiteStorageFile: storage file. |
514 """ | 520 """ |
515 return SQLiteStorageFile(storage_type=self._storage_type) | 521 return SQLiteStorageFile(storage_type=self._storage_type) |
516 | 522 |
517 def _CreateTaskStorageMergeReader(self, path): | 523 def _CreateTaskStorageMergeReader(self, path): |
518 """Creates a task storage merge reader. | 524 """Creates a task storage merge reader. |
519 | 525 |
520 Args: | 526 Args: |
521 path (str): path of the task storage file that should be merged. | 527 path (str): path to the task storage file that should be merged. |
522 | 528 |
523 Returns: | 529 Returns: |
524 StorageMergeReader: storage merge reader. | 530 SQLiteStorageMergeReader: storage merge reader. |
525 """ | 531 """ |
526 return SQLiteStorageMergeReader(self, path) | 532 return SQLiteStorageMergeReader(self, path) |
527 | 533 |
528 def _CreateTaskStorageWriter(self, path, task): | 534 def _CreateTaskStorageWriter(self, path, task): |
529 """Creates a task storage writer. | 535 """Creates a task storage writer. |
530 | 536 |
531 Args: | 537 Args: |
532 path (str): path of the storage file. | 538 path (str): path to the storage file. |
533 task (Task): task. | 539 task (Task): task. |
534 | 540 |
535 Returns: | 541 Returns: |
536 StorageWriter: storage writer. | 542 SQLiteStorageFileWriter: storage writer. |
537 """ | 543 """ |
538 return SQLiteStorageFileWriter( | 544 return SQLiteStorageFileWriter( |
539 self._session, path, | 545 self._session, path, |
540 storage_type=definitions.STORAGE_TYPE_TASK, task=task) | 546 storage_type=definitions.STORAGE_TYPE_TASK, task=task) |
LEFT | RIGHT |