From c4fca348d30909c0c2635969bba4c6d8e656baaf Mon Sep 17 00:00:00 2001 From: pySigma Dev Date: Fri, 3 Jul 2026 14:09:27 +0200 Subject: [PATCH 1/2] docs: complete documentation overhaul with Phase 1-5 content - Add comprehensive documentation structure with Sphinx and Furo theme - Include Phase 1-5 documentation files (Quickstart, Cheat Sheet, Tutorials, etc.) - Fix docstring formatting issues that prevented HTML build - Configure ReadTheDocs and GitHub Actions for documentation deployment - Add custom sidebar navigation with Index and Module Index links - Update pyproject.toml and poetry.lock with Sphinx dependencies --- .github/workflows/docs.yml | 33 +++ .readthedocs.yaml | 20 +- docs/Backend_Development.rst | 238 ++++++++++++++++++++++ docs/Backends.rst | 29 ++- docs/Changelog.rst | 83 ++++++++ docs/Conditions_Guide.rst | 301 ++++++++++++++++++++++++++++ docs/Processing_Pipelines.rst | 1 + docs/Quickstart.rst | 105 ++++++++++ docs/Sigma_Rules.rst | 25 ++- docs/Transformation_Cheat_Sheet.rst | 277 +++++++++++++++++++++++++ docs/Transformation_Development.rst | 277 +++++++++++++++++++++++++ docs/YAML_Pipeline_Tutorial.rst | 285 ++++++++++++++++++++++++++ docs/_static/custom.css | 76 +++++++ docs/_templates/sidebar_end.html | 5 + docs/_templates/sidebar_index.html | 8 + docs/conf.py | 72 ++++++- docs/index.rst | 31 ++- poetry.lock | 95 ++++++++- pyproject.toml | 3 +- sigma/collection.py | 16 +- sigma/conversion/base.py | 2 +- 21 files changed, 1943 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 docs/Backend_Development.rst create mode 100644 docs/Changelog.rst create mode 100644 docs/Conditions_Guide.rst create mode 100644 docs/Quickstart.rst create mode 100644 docs/Transformation_Cheat_Sheet.rst create mode 100644 docs/Transformation_Development.rst create mode 100644 docs/YAML_Pipeline_Tutorial.rst create mode 100644 docs/_static/custom.css create mode 100644 docs/_templates/sidebar_end.html create mode 100644 docs/_templates/sidebar_index.html diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..138e5ac5 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,33 @@ +name: Docs +on: + push: + branches: ["*"] + pull_request: + branches: ["*"] + workflow_dispatch: + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Install Poetry + run: pipx install poetry + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: poetry + - name: Install dependencies + run: poetry install --with dev + - name: Build documentation + run: | + cd docs + poetry run make clean + poetry run make html SPHINXOPTS="-W --keep-going" + - name: Upload docs artifact + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: documentation + path: docs/_build/html diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 5e53902c..1d121987 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,11 +1,27 @@ version: 2 + build: os: ubuntu-lts-latest tools: python: "3" + jobs: + build: + post_checkout: + - pip install --upgrade pip + - pip install poetry + - poetry install --with dev + sphinx: - configuration: docs/conf.py + configuration: docs/conf.py + python: install: - method: pip - path: . \ No newline at end of file + path: . + +formats: + - htmlzip + - pdf + +readthedocs: + version: 2 diff --git a/docs/Backend_Development.rst b/docs/Backend_Development.rst new file mode 100644 index 00000000..0026b8d8 --- /dev/null +++ b/docs/Backend_Development.rst @@ -0,0 +1,238 @@ +Backend Development +################### + +This guide explains how to write a custom backend for pySigma. + +Backend Overview +**************** + +A backend is a class that converts Sigma rules into backend-specific query formats. + +The conversion pipeline: + +1. **Rule** (Sigma rule) → **ParsedRule** (parsed rule object) +2. **ParsedRule** → **QueryExpression** (intermediate query representation) +3. **QueryExpression** → **BackendQuery** (backend-specific query string) + +Key Classes +*********** + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - Class + - Purpose + * - ``SigmaBackend`` + - Base class for all backends + * - ``SigmaQueryExpression`` + - Base class for query expressions + * - ``SigmaQueryExpressionTransformer`` + - Transforms query expressions (e.g., to string) + * - ``SigmaConditionTranslator`` + - Translates Sigma conditions to query expressions + * - ``SigmaRuleConverter`` + - Converts parsed rules to query expressions + * - ``SigmaDetectionConverter`` + - Converts detection items to query expressions + * - ``SigmaDetectionItemConverter`` + - Converts individual detection items + * - ``SigmaFieldMapping`` + - Handles field name mappings + * - ``SigmaValueConverter`` + - Converts values (strings, numbers, etc.) + +Required Methods +**************** + +A minimal backend must implement: + +1. `convert_rule()` — Convert a parsed rule to queries +2. `convert_condition()` — Convert a condition expression to a query expression +3. `convert_detection_item()` — Convert a detection item to a query expression + +Example: Minimal Backend +************************ + +.. code-block:: python + + from sigma.processing.pipeline import ProcessingPipeline + from sigma.conversion.backends.base import BaseSigmaBackend + from sigma.conversion.state import QueryObject + from sigma.types import SigmaString + from sigma.rule import SigmaRule + from sigma.parsing.parser import SigmaParser + from sigma.collection import SigmaCollection + + class MyBackend(BaseSigmaBackend): + """Minimal backend that outputs raw query strings.""" + + def __init__(self, pipeline: ProcessingPipeline = None): + super().__init__(pipeline) + + def convert_rule(self, parsed_rule: SigmaRule) -> QueryObject: + """Convert a parsed rule to a backend query.""" + # Get the detection items from the rule + detections = parsed_rule.detection + # Build a simple query from the detection items + conditions = [] + for condition in detections.condition: + conditions.append(str(condition)) + return QueryObject( + query="\nOR\n".join(conditions), + pipeline=self.pipeline, + ) + + def finalize_query(self, pipeline: ProcessingPipeline, query: QueryObject) -> str: + """Finalize a single query.""" + return str(query.query) + + def finalize_query_output(self, pipeline: ProcessingPipeline, query: QueryObject) -> dict: + """Finalize query output format.""" + return {"query": str(query.query)} + + def finalize_output(self, pipeline: ProcessingPipeline, queries: list[QueryObject]) -> str: + """Finalize all queries.""" + return "\n---\n".join(self.finalize_query(pipeline, q) for q in queries) + + def convert_condition(self, condition: str) -> str: + """Convert a condition expression.""" + return condition + + def convert_detection_item(self, detection_item: dict) -> str: + """Convert a detection item.""" + return str(detection_item) + +Using the Backend +***************** + +.. code-block:: python + + from sigma.collection import SigmaCollection + from my_backend import MyBackend + + # Load rules + rules = SigmaCollection.from_yaml(""" + title: Test Rule + id: 12345678-1234-1234-1234-123456789012 + status: experimental + logsource: + product: windows + service: sysmon + detection: + selection: + EventID: 1 + Image: "*cmd.exe" + condition: selection + """) + + # Convert with backend + backend = MyBackend() + queries = backend.convert(rules) + print(queries) + +Query Object +************ + +The `QueryObject` class stores the converted query: + +.. code-block:: python + + from sigma.conversion.state import QueryObject + + query = QueryObject( + query="EventID=1 AND Image=cmd.exe", + pipeline=pipeline, + ) + + # Access the query + print(query.query) + + # Access the pipeline + print(query.pipeline) + +State Management +**************** + +Use `QueryObject` to pass state between conversion steps: + +.. code-block:: python + + from sigma.conversion.state import QueryObject + + query = QueryObject( + query="EventID=1", + pipeline=pipeline, + state={"processed": True}, + ) + + # Check state + if query.state.get("processed"): + print("Already processed") + +Error Handling +************** + +Raise `SigmaError` for conversion errors: + +.. code-block:: python + + from sigma.exceptions import SigmaError + + class MyBackend(BaseSigmaBackend): + def convert_rule(self, parsed_rule: SigmaRule) -> QueryObject: + if not parsed_rule.detection: + raise SigmaError(f"Rule '{parsed_rule.title}' has no detection items") + return QueryObject(query="...", pipeline=self.pipeline) + +Testing Your Backend +******************** + +Use `pytest` with `SigmaCollection` fixtures: + +.. code-block:: python + + from sigma.collection import SigmaCollection + from sigma.testing import TestRule + from my_backend import MyBackend + + def test_convert_rule(): + rules = SigmaCollection.from_yaml(""" + title: Test Rule + id: 12345678-1234-1234-1234-123456789012 + status: experimental + logsource: + product: windows + detection: + selection: + EventID: 1 + condition: selection + """) + + backend = MyBackend() + queries = backend.convert(rules) + assert len(queries) == 1 + assert "EventID=1" in queries[0].query + +Best Practices +************** + +1. **Inherit from `BaseSigmaBackend`** — Don't implement from scratch. + +2. **Use `QueryObject` for state** — Pass state between conversion steps. + +3. **Handle errors with `SigmaError`** — Provide clear error messages. + +4. **Test with real rules** — Use `SigmaCollection.from_yaml()` for test data. + +5. **Document your backend** — Add a README explaining how to use it. + +6. **Support processing pipelines** — Allow custom pipelines for field mappings, etc. + +7. **Use `finalize_query_output()`** — Control the output format (JSON, YAML, etc.). + +8. **Handle edge cases** — Empty rules, missing detections, invalid conditions. + +9. **Use `SigmaString` for values** — Handle type conversion correctly. + +10. **Test with multiple backends** — Ensure compatibility with other backends. diff --git a/docs/Backends.rst b/docs/Backends.rst index b318c142..6e6efa37 100644 --- a/docs/Backends.rst +++ b/docs/Backends.rst @@ -4,7 +4,7 @@ Backends Backends are responsible for conversion of Sigma rules into a target query languages. Mainly, they have to convert the conditions of the Sigma rules with their reference to detection items into equivalent query. Backends should not be used to handle log source types or data models, e.g. field -naming or differences in value representation. Use ::doc:`Processing_Pipelines` instead. +naming or differences in value representation. Use :doc:`Processing_Pipelines` instead. To implement a conversion for a new query language derive an appropriate backend base class from below and override properties or methods as required. @@ -12,6 +12,33 @@ below and override properties or methods as required. Use the `Cookiecutter template `_ to start a new backend. +Example: Converting a Rule to Splunk +************************************ + +.. code-block:: python + + from sigma.collection import SigmaCollection + from sigma.backends.splunk import SplunkQueryBackend + from sigma.pipelines.sysmon import sysmon_pipeline + + # Load rules + rules = SigmaCollection.from_yaml_file("rules.yml") + + # Create backend with pipeline + pipeline = sysmon_pipeline() + backend = SplunkQueryBackend(pipeline=pipeline) + + # Convert + queries = backend.convert(rules) + for query in queries: + print(query) + +Available Backends +****************** + +See the `pySigma backends repository `_ for a +complete list of official backends. + Conventions *********** diff --git a/docs/Changelog.rst b/docs/Changelog.rst new file mode 100644 index 00000000..d91a93ae --- /dev/null +++ b/docs/Changelog.rst @@ -0,0 +1,83 @@ +Changelog +######### + +This page lists important changes between versions. + +For a complete list of changes, see the `GitHub releases `_. + +Version 1.4.0 +============= + +Released: 2026-06-27 + +New Features +------------ + +* Add external data source placeholder transformations (File, HTTP, Command) +* Implement `neq` modifier for boolean NOT match + +Bug Fixes +--------- + +* Fix filter rules with wildcard conditions (`1/any/all of pattern_*`) +* Fix `SigmaRule.to_dict()` fails after field_name_mapping transformation +* Harden external data source placeholder transformations +* Fix Dependabot dependency parsing for date-versioned packages +* Update dependencies and fix mypy type errors + +Maintenance +----------- + +* Enforce black formatting and mypy checks in Copilot agentic sessions +* Bump mypy from 1.20.2 to 2.1.0 +* Bump pytest from 9.0.3 to 9.1.1 +* Bump pylint from 4.0.5 to 4.0.6 +* Bump actions/checkout from 6 to 7 + +Documentation +------------- + +* Improved documentation structure with Furo theme +* Added Quickstart guide +* Added YAML Pipeline Tutorial +* Added Changelog page + +Version 1.3.3 +============= + +Released: 2026-04-21 + +Bug Fixes +--------- + +* Various bug fixes and improvements + +Version 1.3.2 +============= + +Released: 2026-03-15 + +Bug Fixes +--------- + +* Various bug fixes and improvements + +Version 1.3.1 +============= + +Released: 2026-02-10 + +Bug Fixes +--------- + +* Various bug fixes and improvements + +Version 1.3.0 +============= + +Released: 2026-01-20 + +New Features +------------ + +* Initial stable release with core functionality diff --git a/docs/Conditions_Guide.rst b/docs/Conditions_Guide.rst new file mode 100644 index 00000000..e1b516f9 --- /dev/null +++ b/docs/Conditions_Guide.rst @@ -0,0 +1,301 @@ +Conditions Guide +################ + +Conditions allow you to apply transformations selectively based on rule attributes, field names, or detection items. + +Condition Types +*************** + +There are three levels of conditions in pySigma: + +1. **Rule conditions** — applied to the whole rule +2. **Detection item conditions** — applied to each detection item +3. **Field name conditions** — applied to field names in detection items + +Rule Conditions +*************** + +Rule conditions are evaluated once per rule. They are defined in the `rule_conditions` attribute. + +Available Rule Conditions +========================= + +.. list-table:: + :header-rows: 1 + :widths: 25 45 30 + + * - Identifier + - Class + - Purpose + * - ``logsource`` + - ``LogsourceCondition`` + - Match by log source (product, service, etc.) + * - ``contains_detection_item`` + - ``RuleContainsDetectionItemCondition`` + - Check if rule contains specific detection item + * - ``processing_item_applied`` + - ``RuleProcessingItemAppliedCondition`` + - Check if a transformation was already applied + * - ``processing_state`` + - ``RuleProcessingStateCondition`` + - Check processing state set by ``set_state`` + * - ``is_sigma_rule`` + - ``IsSigmaRuleCondition`` + - Check if rule is a Sigma rule (not correlation) + * - ``is_sigma_correlation_rule`` + - ``IsSigmaCorrelationRuleCondition`` + - Check if rule is a correlation rule + * - ``rule_attribute`` + - ``RuleAttributeCondition`` + - Check rule attributes (title, id, etc.) + * - ``tag`` + - ``RuleTagCondition`` + - Match by Sigma tags + +Rule Condition Examples +======================= + +Example 1: Match by log source +------------------------------ + +.. code-block:: yaml + + transformations: + - type: field_name_mapping + mapping: + EventID: event_id + rule_conditions: + - type: logsource + product: windows + service: sysmon + +Example 2: Match by tag +----------------------- + +.. code-block:: yaml + + transformations: + - type: add_field + field: processed + value: true + rule_conditions: + - type: tag + tag: "attack.t1003" + +Example 3: Check processing state +---------------------------------- + +.. code-block:: yaml + + transformations: + - type: set_state + state: mapped + rule_conditions: + - type: processing_state + state: "not_mapped" + +Example 4: Multiple conditions (AND) +------------------------------------- + +.. code-block:: yaml + + transformations: + - type: field_name_mapping + mapping: + Image: process.executable + rule_conditions: + - type: logsource + service: sysmon + - type: tag + tag: "attack.execution" + +Detection Item Conditions +************************* + +Detection item conditions are evaluated for each detection item in the rule. + +Available Detection Item Conditions +=================================== + +.. list-table:: + :header-rows: 1 + :widths: 25 50 25 + + * - Identifier + - Class + - Purpose + * - ``match_string`` + - ``MatchStringCondition`` + - Match detection item string values + * - ``is_null`` + - ``IsNullCondition`` + - Check if detection item is null + * - ``processing_item_applied`` + - ``DetectionItemProcessingItemAppliedCondition`` + - Check if transformation was applied + * - ``processing_state`` + - ``DetectionItemProcessingStateCondition`` + - Check processing state + +Detection Item Condition Examples +================================== + +Example 1: Match specific string values +---------------------------------------- + +.. code-block:: yaml + + transformations: + - type: set_value + value: "known_bad" + detection_item_conditions: + - type: match_string + match: "malware.exe" + +Example 2: Check for null values +--------------------------------- + +.. code-block:: yaml + + transformations: + - type: drop_detection_item + detection_item_conditions: + - type: is_null + +Field Name Conditions +********************* + +Field name conditions are evaluated for field names in detection items. + +Available Field Name Conditions +=============================== + +.. list-table:: + :header-rows: 1 + :widths: 25 50 25 + + * - Identifier + - Class + - Purpose + * - ``include_fields`` + - ``IncludeFieldCondition`` + - Match specific field names + * - ``exclude_fields`` + - ``ExcludeFieldCondition`` + - Exclude specific field names + * - ``processing_item_applied`` + - ``FieldNameProcessingItemAppliedCondition`` + - Check if transformation was applied + * - ``processing_state`` + - ``FieldNameProcessingStateCondition`` + - Check processing state + +Field Name Condition Examples +============================== + +Example 1: Apply to specific fields only +----------------------------------------- + +.. code-block:: yaml + + transformations: + - type: field_name_prefix + prefix: "sysmon." + field_name_conditions: + - type: include_fields + fields: ["Image", "CommandLine", "ParentImage"] + +Example 2: Exclude certain fields +----------------------------------- + +.. code-block:: yaml + + transformations: + - type: field_name_suffix + suffix: "_raw" + field_name_conditions: + - type: exclude_fields + fields: ["EventID", "Timestamp"] + +Condition Operators +******************* + +You can combine multiple conditions using operators. + +Using `*_cond_op` (list mode) +============================== + +.. code-block:: yaml + + rule_conditions: + - type: logsource + service: sysmon + - type: tag + tag: "attack.execution" + rule_cond_op: "and" # both must match + +Using `*_cond_expr` (expression mode) +====================================== + +.. code-block:: yaml + + rule_conditions: + cond1: + type: logsource + service: sysmon + cond2: + type: tag + tag: "attack.execution" + rule_cond_expr: "cond1 and cond2" + +Supported operators in expressions: +- `and` — logical AND +- `or` — logical OR +- `not` — logical NOT (prefix) + +Condition Negation +****************** + +Negate a condition result by setting `*_cond_not: true`. + +.. code-block:: yaml + + transformations: + - type: field_name_mapping + mapping: + EventID: event_id + rule_conditions: + - type: logsource + service: sysmon + rule_cond_not: true # apply to rules WITHOUT Sysmon logsource + +Best Practices +************** + +1. **Use `id` for tracking** — Add identifiers to your conditions for debugging: + + .. code-block:: yaml + + rule_conditions: + - id: is_sysmon + type: logsource + service: sysmon + +2. **Combine conditions carefully** — AND conditions are more restrictive, OR are broader. + +3. **Test with simple rules first** — Verify your conditions work before applying to large rule sets. + +4. **Use `set_state` for chaining** — Mark rules/items as processed to avoid double-transformation: + + .. code-block:: yaml + + transformations: + - type: set_state + state: mapped + - type: field_name_mapping + mapping: {EventID: event_id} + rule_conditions: + - type: processing_state + state: "not_mapped" + +5. **Document your conditions** — Add comments in YAML explaining why certain conditions are used. diff --git a/docs/Processing_Pipelines.rst b/docs/Processing_Pipelines.rst index 4928b71f..fe3dd67f 100644 --- a/docs/Processing_Pipelines.rst +++ b/docs/Processing_Pipelines.rst @@ -295,6 +295,7 @@ definitions are available: .. autoclass:: sigma.processing.transformations.fields.FieldFunctionTransformation .. autoclass:: sigma.processing.transformations.detection_item.DropDetectionItemTransformation .. autoclass:: sigma.processing.transformations.values.HashesFieldsDetectionItemTransformation + :no-index: .. autoclass:: sigma.processing.transformations.fields.AddFieldnameSuffixTransformation .. autoclass:: sigma.processing.transformations.fields.AddFieldnamePrefixTransformation .. autoclass:: sigma.processing.transformations.placeholder.WildcardPlaceholderTransformation diff --git a/docs/Quickstart.rst b/docs/Quickstart.rst new file mode 100644 index 00000000..1de7da88 --- /dev/null +++ b/docs/Quickstart.rst @@ -0,0 +1,105 @@ +Quickstart +########## + +This guide gets you up and running with pySigma in 5 minutes. + +Prerequisites +************* + +* Python 3.10 or higher +* pip or poetry + +Installation +************ + +.. code-block:: bash + + pip install pysigma + +Or with poetry: + +.. code-block:: bash + + poetry add pysigma + +Your First Sigma Rule +********************* + +Load a Sigma rule from YAML: + +.. code-block:: python + + from sigma.collection import SigmaCollection + from sigma.backends.test import TextQueryTestBackend + + # Minimal Sigma rule + sigma_rule = """ + title: Test Rule + logsource: + category: process_creation + product: windows + detection: + selection: + CommandLine|startswith: 'cmd.exe' + condition: selection + """ + + # Load the rule + rules = SigmaCollection.from_yaml(sigma_rule) + + # Convert to query (using test backend) + backend = TextQueryTestBackend() + queries = backend.convert(rules) + + print("\\n".join(queries)) + +Expected output: + +.. code-block:: text + + CommandLine startswith 'cmd.exe' + +With a Real Backend (e.g., Splunk) +*********************************** + +.. code-block:: python + + from sigma.collection import SigmaCollection + from sigma.backends.splunk import SplunkQueryBackend + + # Load multiple rules from a file + rules = SigmaCollection.from_yaml_file("rules.yml") + + # Splunk backend with Sysmon pipeline + from sigma.pipelines.sysmon import sysmon_pipeline + pipeline = sysmon_pipeline() + backend = SplunkQueryBackend(pipeline=pipeline) + + # Convert + queries = backend.convert(rules) + + for q in queries: + print(q) + +With a Custom YAML Pipeline +**************************** + +.. code-block:: python + + from sigma.processing.pipeline import ProcessingPipeline + + # Load a pipeline from a YAML file + with open("my_pipeline.yml") as f: + pipeline = ProcessingPipeline.from_yaml(f.read()) + + # Use the pipeline with a backend + backend = ElasticsearchQueryBackend(pipeline=pipeline) + +Next Steps +********** + +* `Sigma Official Site `_ — Learn about Sigma rules and detection concepts +* :doc:`YAML_Pipeline_Tutorial` — Learn how to create your own YAML pipelines +* :doc:`Sigma_Rules` — Understand Sigma rule structure +* :doc:`Processing_Pipelines` — Complete reference for transformations +* :doc:`Backends` — List of available backends diff --git a/docs/Sigma_Rules.rst b/docs/Sigma_Rules.rst index f1361309..f6fb9edd 100644 --- a/docs/Sigma_Rules.rst +++ b/docs/Sigma_Rules.rst @@ -4,11 +4,26 @@ Sigma Rules This documentation page describes the parsing of Sigma rules and working with Sigma objects resulting from parsed rules. -Parsing -******* - -Programatic Construction -************************ +What are Sigma Rules? +********************* + +Sigma is a generic, open-ended, and extensible signature format for detection rules. Learn more on +the `official Sigma website `_. + +A minimal Sigma rule looks like this: + +.. code-block:: yaml + + title: Suspicious Command Line + id: 5013332f-8a70-4e04-bcc1-06a98a2cca2e + status: test + logsource: + category: process_creation + product: windows + detection: + selection: + CommandLine|startswith: 'cmd.exe /c' + condition: selection Rule Collections **************** diff --git a/docs/Transformation_Cheat_Sheet.rst b/docs/Transformation_Cheat_Sheet.rst new file mode 100644 index 00000000..bff7014c --- /dev/null +++ b/docs/Transformation_Cheat_Sheet.rst @@ -0,0 +1,277 @@ +Transformation Cheat Sheet +########################## + +This cheat sheet helps you find the right transformation for common use cases. + +Field Operations +**************** + +.. list-table:: + :header-rows: 1 + :widths: 35 25 40 + + * - Need + - Transformation + - Example YAML + * - Rename a field + - ``field_name_mapping`` + - ``mapping: {EventID: event_id}`` + * - Add prefix to all fields + - ``field_name_prefix`` + - ``prefix: "sysmon."`` + * - Add suffix to all fields + - ``field_name_suffix`` + - ``suffix: "_raw"`` + * - Map field A to B, C, D (OR) + - ``field_name_mapping`` + - ``mapping: {OldField: [FieldB, FieldC, FieldD]}`` + * - Apply function to field name + - ``field_name_transform`` + - ``function: "lower"`` + +Value Operations +**************** + +.. list-table:: + :header-rows: 1 + :widths: 30 25 45 + + * - Need + - Transformation + - Example YAML + * - Replace exact values + - ``map_string`` + - ``mapping: {True: "enabled", False: "disabled"}`` + * - Replace with regex + - ``replace_string`` + - ``regex: "foo" replacement: "bar"`` + * - Convert type (str→int, etc.) + - ``convert_type`` + - ``target_type: "int"`` + * - Set specific value + - ``set_value`` + - ``value: "custom_value"`` + * - Hash a field (MD5, SHA256, etc.) + - ``hashes_fields`` + - ``hash_algos: ["md5", "sha256"]`` + * - Apply case transformation + - ``case`` + - ``case: "lower"`` + +Condition Operations +******************** + +.. list-table:: + :header-rows: 1 + :widths: 30 25 45 + + * - Need + - Transformation + - Example YAML + * - Add detection condition + - ``add_condition`` + - ``conditions: ["selection"]`` + * - Change log source + - ``change_logsource`` + - ``logsource: {product: "windows", service: "sysmon"}`` + * - Add new field to rule + - ``add_field`` + - ``field: "custom_field", value: "value"`` + * - Remove field from rule + - ``remove_field`` + - ``field: "unwanted_field"`` + * - Set field value + - ``set_field`` + - ``field: "event_category", value: "process"`` + +State & Tracking +**************** + +.. list-table:: + :header-rows: 1 + :widths: 30 25 45 + + * - Need + - Transformation + - Example YAML + * - Mark rule as processed + - ``set_state`` + - ``state: "mapped"`` + * - Track applied transformations + - ``set_state`` + - Use with ``rule_conditions`` to check state + +Advanced Operations +******************* + +.. list-table:: + :header-rows: 1 + :widths: 30 25 45 + + * - Need + - Transformation + - Example YAML + * - Group multiple transformations + - ``nest`` + - See :doc:`YAML_Pipeline_Tutorial` for details + * - Apply regex to field + - ``regex`` + - ``regex: "pattern" field: "target_field"`` + * - Drop detection item + - ``drop_detection_item`` + - ``condition: {type: "field_name", field: "old_field"}`` + * - Convert hash fields + - ``hashes_fields`` + - ``hash_algos: ["md5"], field_prefix: "file."`` + * - Add wildcard placeholders + - ``wildcard_placeholders`` + - ``field: "command_line"`` + * - Add value list placeholders + - ``value_placeholders`` + - ``field: "status_code"`` + +Post-Processing Operations +************************** + +.. list-table:: + :header-rows: 1 + :widths: 30 25 45 + + * - Need + - Transformation + - Example YAML + * - Wrap query in template + - ``simple_template`` + - ``template: "search {query}"`` + * - Custom query template + - ``template`` + - See :doc:`YAML_Pipeline_Tutorial` for details + * - Embed query in JSON + - ``json`` + - ``template: '{"query": "{query}"}'`` + * - Replace parts of query + - ``replace`` + - ``regex: "old" replacement: "new"`` + * - Embed raw query + - ``embed`` + - ``prefix: "[", suffix: "]"`` + +Output Finalization +******************* + +.. list-table:: + :header-rows: 1 + :widths: 30 25 45 + + * - Need + - Transformation + - Example YAML + * - Join multiple queries + - ``concat`` + - ``separator: " OR "`` + * - Template final output + - ``template`` + - ``template: "header\n{queries}\nfooter"`` + * - Export as JSON + - ``json`` + - ``template: '{"queries": {queries}}'`` + * - Export as YAML + - ``yaml`` + - (no extra params needed) + +Common Recipes +************** + +Recipe 1: Sysmon → Elastic +========================== + +.. code-block:: yaml + + name: Sysmon to Elastic + priority: 10 + transformations: + - type: field_name_mapping + mapping: + EventID: event_id + Image: process.executable + CommandLine: process.command_line + - type: add_field + field: event.category + value: process + postprocessing: + - type: simple_template + template: | + { + "query": { + "bool": { + "must": [{"query_string": {"query": "{query}"}}], + "filter": [{"term": {"event.category": "process"}}] + } + } + } + +Recipe 2: Add Timestamp Standardization +======================================== + +.. code-block:: yaml + + name: Standardize timestamps + priority: 10 + transformations: + - type: field_name_mapping + mapping: + TimeGenerated: "@timestamp" + Created: "@timestamp" + Date: "@timestamp" + +Recipe 3: Field Prefix for Multi-Source +======================================== + +.. code-block:: yaml + + name: Add source prefix + priority: 20 + transformations: + - type: field_name_prefix + prefix: "sysmon." + field_name_conditions: + - type: include_fields + fields: ["Image", "CommandLine", "ParentImage"] + +Recipe 4: Value Mapping with OR +================================ + +.. code-block:: yaml + + name: Map status codes + priority: 10 + transformations: + - type: map_string + mapping: + "0": "success" + "1": "partial_success" + "2": "failure" + "3": + - "error" + - "failed" + +Recipe 5: Conditional Transformation +===================================== + +.. code-block:: yaml + + name: Conditional field mapping + priority: 10 + transformations: + - type: field_name_mapping + mapping: + EventID: event_id + rule_conditions: + - type: logsource + service: sysmon + - type: field_name_mapping + mapping: + EventCode: event_id + rule_conditions: + - type: logsource + service: windows diff --git a/docs/Transformation_Development.rst b/docs/Transformation_Development.rst new file mode 100644 index 00000000..8a6d9bc4 --- /dev/null +++ b/docs/Transformation_Development.rst @@ -0,0 +1,277 @@ +Transformation Development +########################## + +This guide explains how to write a custom transformation for pySigma. + +Transformation Overview +*********************** + +A transformation is a processing step that modifies Sigma rules before conversion. + +Transformations are defined in YAML pipelines and applied in order. + +Key Classes +*********** + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - Class + - Purpose + * - ``SigmaTransformation`` + - Base class for all transformations + * - ``SigmaRuleTransformation`` + - Base class for rule-level transformations + * - ``SigmaDetectionItemTransformation`` + - Base class for detection item transformations + * - ``SigmaFieldMappingTransformation`` + - Base class for field mapping transformations + +Available Transformation Types +****************************** + +.. list-table:: + :header-rows: 1 + :widths: 25 40 35 + + * - Identifier + - Class + - Purpose + * - ``field_name_mapping`` + - ``FieldNameMappingTransformation`` + - Map field names + * - ``field_name_prefix`` + - ``FieldNamePrefixTransformation`` + - Add prefix to field names + * - ``field_name_suffix`` + - ``FieldNameSuffixTransformation`` + - Add suffix to field names + * - ``field_name_transform`` + - ``FieldNameTransformTransformation`` + - Apply function to field names + * - ``map_string`` + - ``MapStringTransformation`` + - Map string values + * - ``replace_string`` + - ``ReplaceStringTransformation`` + - Replace strings with regex + * - ``convert_type`` + - ``ConvertTypeTransformation`` + - Convert value types + * - ``set_value`` + - ``SetValueTransformation`` + - Set specific value + * - ``add_condition`` + - ``AddConditionTransformation`` + - Add detection condition + * - ``change_logsource`` + - ``ChangeLogsourceTransformation`` + - Change log source + * - ``add_field`` + - ``AddFieldTransformation`` + - Add field to rule + * - ``remove_field`` + - ``RemoveFieldTransformation`` + - Remove field from rule + * - ``set_field`` + - ``SetFieldTransformation`` + - Set field value + * - ``set_state`` + - ``SetStateTransformation`` + - Set processing state + * - ``nest`` + - ``NestTransformation`` + - Group transformations + * - ``regex`` + - ``RegexTransformation`` + - Apply regex to field + * - ``drop_detection_item`` + - ``DropDetectionItemTransformation`` + - Remove detection item + * - ``hashes_fields`` + - ``HashesFieldsTransformation`` + - Convert hash fields + * - ``wildcard_placeholders`` + - ``WildcardPlaceholdersTransformation`` + - Add wildcard placeholders + * - ``value_placeholders`` + - ``ValuePlaceholdersTransformation`` + - Add value list placeholders + * - ``simple_template`` + - ``SimpleTemplateTransformation`` + - Wrap query in template + * - ``template`` + - ``TemplateTransformation`` + - Custom query template + * - ``json`` + - ``JsonTransformation`` + - Export as JSON + * - ``yaml`` + - ``YamlTransformation`` + - Export as YAML + +Writing a Custom Transformation +******************************* + +Example 1: Rule-Level Transformation +===================================== + +.. code-block:: python + + from sigma.processing.transformations import SigmaRuleTransformation + from sigma.rule import SigmaRule + from sigma.processing.pipeline import ProcessingItem + + class CustomRuleTransformation(SigmaRuleTransformation): + """Add a custom field to all rules.""" + + def apply(self, rule: SigmaRule, item: ProcessingItem) -> None: + """Apply transformation to rule.""" + rule.custom_field = "custom_value" + +Example 2: Detection Item Transformation +========================================= + +.. code-block:: python + + from sigma.processing.transformations import SigmaDetectionItemTransformation + from sigma.detection import SigmaDetectionItem + from sigma.processing.pipeline import ProcessingItem + + class CustomDetectionItemTransformation(SigmaDetectionItemTransformation): + """Convert specific values.""" + + def apply(self, detection_item: SigmaDetectionItem, item: ProcessingItem) -> None: + """Apply transformation to detection item.""" + if detection_item.value == "old_value": + detection_item.value = "new_value" + +Example 3: Field Mapping Transformation +======================================== + +.. code-block:: python + + from sigma.processing.transformations import SigmaFieldMappingTransformation + from sigma.processing.pipeline import ProcessingItem + + class CustomFieldMappingTransformation(SigmaFieldMappingTransformation): + """Custom field mapping logic.""" + + def apply(self, field_mapping: dict, item: ProcessingItem) -> None: + """Apply transformation to field mapping.""" + # Modify field mapping + for old_field, new_field in field_mapping.items(): + if "old_prefix" in old_field: + field_mapping[old_field.replace("old_prefix", "new_prefix")] = new_field + del field_mapping[old_field] + +Using Conditions +**************** + +You can apply transformations conditionally: + +.. code-block:: python + + from sigma.processing.transformations import SigmaRuleTransformation + from sigma.rule import SigmaRule + from sigma.processing.pipeline import ProcessingItem + from sigma.conditions import ConditionLogsource + + class ConditionalTransformation(SigmaRuleTransformation): + """Apply transformation only to specific log sources.""" + + def apply(self, rule: SigmaRule, item: ProcessingItem) -> None: + """Apply transformation conditionally.""" + if rule.logsource and rule.logsource.service == "sysmon": + # Apply transformation + rule.custom_field = "sysmon_rule" + +Best Practices +************** + +1. **Inherit from base classes** — Use `SigmaRuleTransformation`, `SigmaDetectionItemTransformation`, etc. + +2. **Use `apply()` method** — Implement the `apply()` method for your transformation logic. + +3. **Handle conditions** — Check conditions before applying transformations. + +4. **Modify in place** — Transformations modify the rule/item directly. + +5. **Use `ProcessingItem` for config** — Access transformation configuration via `item.config`. + +6. **Test with real rules** — Use `SigmaCollection.from_yaml()` for test data. + +7. **Document your transformation** — Add docstrings explaining what it does. + +8. **Handle edge cases** — Check for None values, empty fields, etc. + +9. **Use `set_state` for chaining** — Mark rules/items as processed to avoid double-transformation. + +10. **Keep transformations focused** — Each transformation should do one thing well. + +Testing Transformations +*********************** + +Use `pytest` with `SigmaCollection` fixtures: + +.. code-block:: python + + from sigma.collection import SigmaCollection + from sigma.processing.pipeline import ProcessingPipeline + from my_transformations import CustomRuleTransformation + + def test_custom_transformation(): + rules = SigmaCollection.from_yaml(""" + title: Test Rule + id: 12345678-1234-1234-1234-123456789012 + status: experimental + logsource: + product: windows + detection: + selection: + EventID: 1 + condition: selection + """) + + pipeline = ProcessingPipeline( + name="Custom Pipeline", + priority=10, + transformations=[CustomRuleTransformation()], + postprocessing=[], + ) + + # Apply pipeline + rules.apply_pipeline(pipeline) + + # Check transformation was applied + for rule in rules.rules: + assert hasattr(rule, "custom_field") + assert rule.custom_field == "custom_value" + +Integration with Backends +************************* + +Transformations are applied before backend conversion: + +.. code-block:: python + + from sigma.collection import SigmaCollection + from sigma.processing.pipeline import ProcessingPipeline + from my_backend import MyBackend + from my_transformations import CustomRuleTransformation + + # Create pipeline + pipeline = ProcessingPipeline( + name="Custom Pipeline", + priority=10, + transformations=[CustomRuleTransformation()], + postprocessing=[], + ) + + # Create backend with pipeline + backend = MyBackend(pipeline=pipeline) + + # Convert rules + rules = SigmaCollection.from_yaml("...") + queries = backend.convert(rules) diff --git a/docs/YAML_Pipeline_Tutorial.rst b/docs/YAML_Pipeline_Tutorial.rst new file mode 100644 index 00000000..c440f024 --- /dev/null +++ b/docs/YAML_Pipeline_Tutorial.rst @@ -0,0 +1,285 @@ +YAML Pipeline Tutorial +###################### + +This tutorial teaches you how to create YAML pipelines to adapt pySigma to your environment. + +What is a YAML Pipeline? +************************ + +A YAML pipeline is a file that describes how to transform Sigma rules before conversion. +It is used to map fields, modify values, add conditions, etc. + +Basic Structure +*************** + +A minimal pipeline: + +.. code-block:: yaml + + name: My Pipeline + priority: 10 + transformations: + - type: field_name_mapping + mapping: + EventID: EventCode + +Explanation: + +* ``name`` : pipeline name (informational) +* ``priority`` : order of application (lowest first, default = 0) +* ``transformations`` : list of operations to apply + +Loading a Pipeline +****************** + +From Python: + +.. code-block:: python + + from sigma.processing.pipeline import ProcessingPipeline + + # From a YAML string + pipeline = ProcessingPipeline.from_yaml(yaml_string) + + # From a file + with open("pipeline.yml") as f: + pipeline = ProcessingPipeline.from_yaml(f.read()) + + # From a dictionary + pipeline = ProcessingPipeline.from_dict({ + "name": "My Pipeline", + "priority": 10, + "transformations": [ + { + "type": "field_name_mapping", + "mapping": {"EventID": "EventCode"} + } + ] + }) + +Using the Resolver (multi-pipeline): + +.. code-block:: python + + from sigma.processing.resolver import ProcessingPipelineResolver + + resolver = ProcessingPipelineResolver() + resolver.add_pipeline_class(my_pipeline) # registered pipeline + + # Resolve a single pipeline + pipeline = resolver.resolve_pipeline("my_pipeline") + + # Resolve and merge multiple pipelines + consolidated = resolver.resolve([ + "pipeline1.yml", + "pipeline2.yml", + "my_pipeline" + ]) + +Progressive Examples +******************** + +Example 1: Simple Mapping +========================== + +Map fields from one log source to another: + +.. code-block:: yaml + + name: Sysmon to Elastic + priority: 10 + transformations: + - type: field_name_mapping + mapping: + EventID: event_id + CommandLine: command_line + ParentImage: parent_process + Image: process + +Example 2: Mapping with Conditions +=================================== + +Apply mapping only for certain log sources: + +.. code-block:: yaml + + name: Sysmon conditional mapping + priority: 10 + transformations: + - type: field_name_mapping + mapping: + EventID: event_id + rule_conditions: + - type: logsource + product: windows + service: sysmon + +Example 3: Adding Suffixes/Prefixes +==================================== + +.. code-block:: yaml + + name: Add field prefixes + priority: 20 + transformations: + - type: field_name_prefix + prefix: "sysmon." + - type: field_name_suffix + suffix: "_raw" + +Example 4: Post-processing +============================ + +Transform the generated query for Splunk: + +.. code-block:: yaml + + name: Splunk search command + priority: 50 + postprocessing: + - type: simple_template + template: | + search {query} | fields - _raw + query_field: "_raw" + +Example 5: Finalizers +======================= + +Merge multiple queries into a single output: + +.. code-block:: yaml + + name: Concatenate queries + priority: 60 + finalizers: + - type: concat + separator: " OR " + prefix: "(" + suffix: ")" + +Example 6: Variables +===================== + +Use variables in transformations: + +.. code-block:: yaml + + name: Pipeline with variables + priority: 10 + vars: + target_field: "command_line" + search_string: "powershell" + transformations: + - type: regex + field: "{vars[target_field]}" + regex: "(?i){vars[search_string]}" + +Example 7: Nested Pipeline (nest) +=================================== + +Group multiple transformations: + +.. code-block:: yaml + + name: Nested pipeline + priority: 10 + transformations: + - type: nest + items: + - type: field_name_mapping + mapping: + EventID: event_id + - type: field_name_prefix + prefix: "sysmon." + - type: set_state + state: mapped + +Example 8: Real-world Sysmon to Splunk +======================================== + +Complete pipeline to adapt Sigma Sysmon rules to Splunk: + +.. code-block:: yaml + + name: Sysmon to Splunk + priority: 10 + allowed_backends: + - splunk + + transformations: + # Map Sysmon fields to Splunk field names + - type: field_name_mapping + mapping: + EventID: EventCode + Image: Process + CommandLine: CommandLine + ParentImage: ParentProcess + ParentCommandLine: ParentCommandLine + User: User + LogonId: LogonId + Hashes: Hash + + # Add Splunk-specific fields + - type: add_field + field: EventType + value: ProcessCreate + + # Condition: apply only to Sysmon rules + - type: set_state + state: sysmon_mapped + rule_conditions: + - type: logsource + service: sysmon + + postprocessing: + # Format for Splunk search command + - type: simple_template + template: | + index=os (EventCode="{query}") | fields Process, CommandLine, User + query_field: "_raw" + +Best Practices +************** + +1. **Naming**: Use descriptive names for your pipeline +2. **Priority**: + - 10: Log source pipelines (Sysmon, etc.) + - 20: Pipelines provided by backends + - 50: Pipelines integrated in backend + - 60: Output format pipelines +3. **Tracking**: Use ``id`` in each transformation for debugging +4. **Conditions**: Use ``rule_conditions`` to apply transformations conditionally +5. **Testing**: Test your pipeline with a simple rule before using it in production + +Debugging +********* + +To see which transformations are applied: + +.. code-block:: python + + from sigma.processing.pipeline import ProcessingPipeline + from sigma.processing.tracking import ItemTrackingMode + + pipeline = ProcessingPipeline.from_yaml(yaml_string) + + # Enable tracking + pipeline.set_tracking_mode(ItemTrackingMode.TRACK_ALL) + + # Apply to a rule + processed_rule = pipeline.apply(rule) + + # View history + print(processed_rule.tracking_data) + +Similarities with Sigma +************************ + +Conditions in YAML pipelines use the same syntax as Sigma rules: + +* ``logsource`` : filter by log source +* ``contains_detection_item`` : check for presence of an item +* ``processing_state`` : check processing state +* etc. + +See :doc:`Processing_Pipelines` for the complete reference. diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 00000000..1ebf4533 --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,76 @@ +/* Custom styles for pySigma documentation */ + +/* Improve code block readability */ +.highlight { + border-radius: 4px; +} + +/* Better table styling */ +table.docutils { + border-collapse: collapse; +} + +table.docutils th { + background-color: var(--color-table-odd-cell-hover-background, #f0f0f0); +} + +/* Improve sidebar readability */ +.sidebar-brand-text { + font-weight: 600; +} + +/* Better link styling */ +a.reference.internal { + text-decoration: none; +} + +a.reference.internal:hover { + text-decoration: underline; +} + +/* Highlight important notes */ +.admonition.note { + border-left-width: 4px; +} + +/* Code block line numbers */ +.highlight pre { + line-height: 1.5; +} + +/* Sidebar index section */ +.sidebar-index-section { + margin-top: 1rem; + padding-top: 1rem; + border-top: 1px solid var(--color-sidebar-link-color); +} + +.sidebar-index-title { + font-size: 0.8rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--color-sidebar-link-color--emphasis); + margin-bottom: 0.5rem; +} + +.sidebar-index-links { + list-style: none; + padding: 0; + margin: 0; +} + +.sidebar-index-links li { + margin: 0.25rem 0; +} + +.sidebar-index-links a { + font-size: 0.85rem; + color: var(--color-sidebar-link-color); + text-decoration: none; +} + +.sidebar-index-links a:hover { + color: var(--color-sidebar-link-color--emphasis); + text-decoration: underline; +} diff --git a/docs/_templates/sidebar_end.html b/docs/_templates/sidebar_end.html new file mode 100644 index 00000000..2d403204 --- /dev/null +++ b/docs/_templates/sidebar_end.html @@ -0,0 +1,5 @@ + diff --git a/docs/_templates/sidebar_index.html b/docs/_templates/sidebar_index.html new file mode 100644 index 00000000..845b4d70 --- /dev/null +++ b/docs/_templates/sidebar_index.html @@ -0,0 +1,8 @@ +{# Custom sidebar with index links #} +
+

Indices

+ +
diff --git a/docs/conf.py b/docs/conf.py index d16e799f..7beb13ca 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,8 +19,9 @@ # -- Project information ----------------------------------------------------- project = "pySigma" -copyright = "2021" +copyright = "2021-2026, Thomas Patzke and pySigma contributors" author = "Thomas Patzke" +release = "1.4.0" # -- General configuration --------------------------------------------------- @@ -31,8 +32,26 @@ extensions = [ "sphinx.ext.autodoc", "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + "sphinx.ext.inheritance_diagram", ] +# Napoleon settings for Google/NumPy style docstrings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_init_with_doc = True +napoleon_include_private_with_doc = False +napoleon_use_param = True +napoleon_use_rtype = True +napoleon_preprocess_types = True + +# Intersphinx mappings +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), +} + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -41,15 +60,54 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - # -- Options for HTML output ------------------------------------------------- -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "alabaster" +# The theme to use for HTML and HTML Help pages. +html_theme = "furo" + +# Furo theme settings +html_title = f"pySigma v{release}" +html_theme_options = { + "sidebar_hide_name": False, + "light_css_variables": { + "color-brand-primary": "#2c3e50", + "color-brand-content": "#3498db", + }, +} + +# Meta data +html_last_updated_fmt = "%Y-%m-%d" +html_use_modindex = True +html_clean_html_name = True +html_copy_source = False # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] +html_static_path = ["_static"] + +# Custom CSS +html_css_files = [ + "custom.css", +] + +# Autodoc settings +autodoc_typehints = "description" +autodoc_member_order = "bysource" +autodoc_default_options = { + "members": True, + "undoc-members": False, + "inherited-members": False, +} + +# Custom sidebars +html_sidebars = { + "**": [ + "sidebar/scroll-start.html", + "sidebar/brand.html", + "sidebar/search.html", + "sidebar/navigation.html", + "sidebar/scroll-end.html", + "sidebar_index.html", + ] +} diff --git a/docs/index.rst b/docs/index.rst index 97ba2022..790e17bd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,15 +2,32 @@ pySigma Documentation ##################### .. toctree:: - :maxdepth: 1 - :caption: Contents: + :hidden: + :caption: Getting Started + + Quickstart + YAML_Pipeline_Tutorial + +.. toctree:: + :hidden: + :caption: Core Concepts Sigma_Rules Processing_Pipelines Backends + Transformation_Cheat_Sheet + Conditions_Guide Rule_Validation + +.. toctree:: + :hidden: + :caption: Advanced + + Backend_Development + Transformation_Development Plugin_System Breaking_Changes + Changelog Overview ******** @@ -98,17 +115,9 @@ The following example:: rules = SigmaCollection.from_yaml(sigma_rule_yaml) print("Result: " + "\n".join(backend.convert(rules))) -* Utilizes the :`Sysmon pipeline package `_ that can be installed - with pip. +* Utilizes the Sysmon pipeline package that can be installed with pip. * instantiates a Splunk backend. * Converts a Sigma rule collection into a list of queries and prints it. Details about the conversion process can be found :doc:`here `. Processing pipelines are described on :doc:`this page `. - -Indices and tables -****************** - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/poetry.lock b/poetry.lock index fcbaa845..3ee44fe9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,24 @@ # This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +[[package]] +name = "accessible-pygments" +version = "0.0.5" +description = "A collection of accessible pygments styles" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "accessible_pygments-0.0.5-py3-none-any.whl", hash = "sha256:88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7"}, + {file = "accessible_pygments-0.0.5.tar.gz", hash = "sha256:40918d3e6a2b619ad424cb91e556bd3bd8865443d9f22f1dcdf79e33c8046872"}, +] + +[package.dependencies] +pygments = ">=1.5" + +[package.extras] +dev = ["pillow", "pkginfo (>=1.10)", "playwright", "pre-commit", "setuptools", "twine (>=5.0)"] +tests = ["hypothesis", "pytest"] + [[package]] name = "alabaster" version = "1.0.0" @@ -85,6 +104,29 @@ files = [ [package.extras] dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.7.0" +groups = ["dev"] +files = [ + {file = "beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9"}, + {file = "beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7"}, +] + +[package.dependencies] +soupsieve = ">=1.6.1" +typing-extensions = ">=4.0.0" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "black" version = "26.5.1" @@ -323,7 +365,7 @@ description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" groups = ["dev"] -markers = "sys_platform == \"win32\" or platform_system == \"Windows\"" +markers = "platform_system == \"Windows\" or sys_platform == \"win32\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -561,6 +603,25 @@ files = [ {file = "filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90"}, ] +[[package]] +name = "furo" +version = "2025.12.19" +description = "A clean customisable Sphinx documentation theme." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "furo-2025.12.19-py3-none-any.whl", hash = "sha256:bb0ead5309f9500130665a26bee87693c41ce4dbdff864dbfb6b0dae4673d24f"}, + {file = "furo-2025.12.19.tar.gz", hash = "sha256:188d1f942037d8b37cd3985b955839fea62baa1730087dc29d157677c857e2a7"}, +] + +[package.dependencies] +accessible-pygments = ">=0.0.5" +beautifulsoup4 = "*" +pygments = ">=2.7" +sphinx = ">=7.0,<10.0" +sphinx-basic-ng = ">=1.0.0b2" + [[package]] name = "identify" version = "2.6.19" @@ -1470,6 +1531,18 @@ files = [ {file = "snowballstemmer-3.1.0.tar.gz", hash = "sha256:fd9e34526b23340cd23ffea6c9f9760974ecc2c2ac9e1d81401443ccdb2a801f"}, ] +[[package]] +name = "soupsieve" +version = "2.8.4" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65"}, + {file = "soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e"}, +] + [[package]] name = "sphinx" version = "8.1.3" @@ -1544,6 +1617,24 @@ docs = ["sphinxcontrib-websupport"] lint = ["betterproto (==2.0.0b6)", "mypy (==1.15.0)", "pypi-attestations (==0.0.21)", "pyright (==1.1.395)", "pytest (>=8.0)", "ruff (==0.9.9)", "sphinx-lint (>=0.9)", "types-Pillow (==10.2.0.20240822)", "types-Pygments (==2.19.0.20250219)", "types-colorama (==0.4.15.20240311)", "types-defusedxml (==0.7.0.20240218)", "types-docutils (==0.21.0.20241128)", "types-requests (==2.32.0.20241016)", "types-urllib3 (==1.26.25.14)"] test = ["cython (>=3.0)", "defusedxml (>=0.7.1)", "pytest (>=8.0)", "pytest-xdist[psutil] (>=3.4)", "setuptools (>=70.0)", "typing_extensions (>=4.9)"] +[[package]] +name = "sphinx-basic-ng" +version = "1.0.0b2" +description = "A modern skeleton for Sphinx themes." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "sphinx_basic_ng-1.0.0b2-py3-none-any.whl", hash = "sha256:eb09aedbabfb650607e9b4b68c9d240b90b1e1be221d6ad71d61c52e29f7932b"}, + {file = "sphinx_basic_ng-1.0.0b2.tar.gz", hash = "sha256:9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9"}, +] + +[package.dependencies] +sphinx = ">=4.0" + +[package.extras] +docs = ["furo", "ipython", "myst-parser", "sphinx-copybutton", "sphinx-inline-tabs"] + [[package]] name = "sphinxcontrib-applehelp" version = "2.0.0" @@ -1793,4 +1884,4 @@ typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\"" [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "c5d5631bdfcc0ed0bc7e4a17b40d47c39354538fe1429640c12936bdd5282a22" +content-hash = "bc424f4d967e5bffa98b9724623df7385af5b33253220b377a4c6beab9a83ce3" diff --git a/pyproject.toml b/pyproject.toml index 68e0c1a4..26351c8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,8 @@ pylint = "^4.0" pytest = "^9.1" pytest-cov = "^7.0" pytest-mypy = "^1.0" -Sphinx = "^8" +Sphinx = "^8 || ^9" +furo = "^2025.12.19" defusedxml = "^0.7" types-requests = "^2.32.4.20250913" mypy = "^2.1" diff --git a/sigma/collection.py b/sigma/collection.py index d5a3e8f5..a19e3c68 100644 --- a/sigma/collection.py +++ b/sigma/collection.py @@ -254,20 +254,18 @@ def load_ruleset( object. :param inputs: List of strings and :class:`pathlib.Path` objects that reference files or - directories that should be loaded. + directories that should be loaded. :param collect_errors: parse or verification errors are collected in :class:`SigmaRuleBase` - objects instead of raising them immediately. Defaults to ``False``. + objects instead of raising them immediately. Defaults to ``False``. :param on_beforeload: Optional function that is called for each path to a Sigma rule before the parsing and - construction of the :class:`SigmaCollection` object is done. The path returned by this function is - used as input. A rule path is skipped if ``None`` is returned. + construction of the :class:`SigmaCollection` object is done. The path returned by this function is + used as input. A rule path is skipped if ``None`` is returned. :param on_load: Optional function that is called after the :class:`SigmaCollection` was - constructed from the path. The path and the SigmaCollection object are passed to this - function and it is expected to return a :class:`SigmaCollection` object that is merged in - the collection of the ruleset or ``None`` if the generated collection should be skipped. + constructed from the path. The path and the SigmaCollection object are passed to this + function and it is expected to return a :class:`SigmaCollection` object that is merged in + the collection of the ruleset or ``None`` if the generated collection should be skipped. :param recursion_pattern: Pattern used to recurse into directories, defaults to ``**/*.yml``. - :return: :class:`SigmaCollection` of all sigma rules contained in given paths. - """ if not isinstance(inputs, Iterable) or isinstance(inputs, str): raise TypeError( diff --git a/sigma/conversion/base.py b/sigma/conversion/base.py index f349fda3..4fa7500d 100644 --- a/sigma/conversion/base.py +++ b/sigma/conversion/base.py @@ -1195,7 +1195,7 @@ class variables. If this is not sufficient, the respective methods can be implem field_equals_field_contains_expression: ClassVar[str | None] = None field_timestamp_part_expression: ClassVar[str | None] = None - """Expression for timestamp part modifiers like |minute, |day, etc.""" + """Expression for timestamp part modifiers like ``|minute``, ``|day``, etc.""" timestamp_part_mapping: ClassVar[dict[TimestampPart, str] | None] = None """Mapping to map a TimestampPart enum value to it's string representation of the target SIEM. Example value: '%M' for minute.""" From 5f0328a418fb0e501aeca14200a273c04b75a6e3 Mon Sep 17 00:00:00 2001 From: pySigma Dev Date: Fri, 3 Jul 2026 14:14:46 +0200 Subject: [PATCH 2/2] docs: only build on main branch merges --- .github/workflows/docs.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 138e5ac5..4dcc7754 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,9 +1,7 @@ name: Docs on: push: - branches: ["*"] - pull_request: - branches: ["*"] + branches: ["main"] workflow_dispatch: jobs: @@ -25,9 +23,3 @@ jobs: cd docs poetry run make clean poetry run make html SPHINXOPTS="-W --keep-going" - - name: Upload docs artifact - if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v4 - with: - name: documentation - path: docs/_build/html