V3.2

Dockerfile

@@ -13,4 +13,4 @@ RUN pip install --upgrade pip setuptools wheel && \
     pip install . --use-feature=in-tree-build
 
 
-ENTRYPOINT [ "python" ]
+ENTRYPOINT [ "python" ]

Makefile

@@ -11,7 +11,7 @@ docker:
 # 	uvicorn salver.api.main:app --reload
 
 engine:
-	docker-compose up
+	docker-compose up --force-recreate
 
 agent:
 	ENV_FOR_DYNACONF=development python -m salver.agent.app
@@ -25,7 +25,7 @@ install:
 format:
 	tox -e black
 	tox -e isort
-	pre-commit run --all-files
+	pre-commit run --all-files -v
 
 bootstrap:
 	./scripts/wait_services_up.sh
@@ -38,3 +38,9 @@ bootstrap:
 
 sloc:
 	pygount --format=summary ./salver
+
+clean:
+	docker-compose -f ./deploy/docker-compose-engine.yml -f ./docker-compose.yml down -v
+	docker container prune -f
+	docker network prune -f
+	docker volume prune -f

README.md

@@ -34,4 +34,11 @@ connectors: neo4J/eladstic make a caching mechanism usique a Q and sched() to fl
 - lister lesscan
 - creer un scan (comme dans toto.py)
 
-- 
+-
+
+
+# PORTS
+
+- Mongo GUI: http://locahost:8017
+- Kafka GUI: http://localhost:8000/
+- Kibana: http://localhost:5601

deploy/docker-compose-engine.yml

@@ -313,6 +313,7 @@ services:
 
 ################################
 # KAFKA TOPICS UI
+
 ################################
     kafka-topics-ui:
         image: landoop/kafka-topics-ui:0.9.4

deploy/elasticsearch/elasticsearch.yml

@@ -1,2 +1,3 @@
-cluster.name: "salver-cluster"
+---
+cluster.name: salver-cluster
 network.host: 0.0.0.0

docker-compose.yml

@@ -2,8 +2,8 @@
 version: '3.5'
 
 networks:
-  deploy_engine:
-    external: true
+    deploy_engine:
+        external: true
 
 services:
 #######################
@@ -16,7 +16,7 @@ services:
         environment:
             ENV_FOR_DYNACONF: development
         networks:
-          - deploy_engine
+            - deploy_engine
         command: -m salver.connectors.app
 
 ########################
@@ -29,5 +29,5 @@ services:
         environment:
             ENV_FOR_DYNACONF: development
         networks:
-          - deploy_engine
-        command: -m salver.engine.app
+            - deploy_engine
+        command: -m salver.engine.app

salver/agent/app.py

@@ -22,7 +22,11 @@ class SalverAgent:
     def __init__(self):
         self.name = f'agent-{socket.getfqdn()}-{uuid4().hex[:4]}'
         all_collectors = [
-            models.Collector(name=c_name, enabled=c_config['enabled'], allowed_input=c_config['allowed_input'])
+            models.Collector(
+                name=c_name,
+                enabled=c_config['enabled'],
+                allowed_input=c_config['allowed_input'],
+            )
             for c_name, c_config in ALL_COLLECTORS.items()
         ]
         enabled_collectors = {
@@ -80,7 +84,7 @@ def start(self):
                 for consumer in self.consumers:
                     consumer.start_workers()
         except KeyboardInterrupt:
-            logger.warning('quitting')
+            logger.warning('Quitting, keyboard interrupt')
             agent_disconnect.produce(self.agent_info, flush=True)
 
 

salver/agent/collectors/dummy/dummy_docker.py

@@ -25,5 +25,5 @@ def cb_email(self, email):
         date = self.run_container(command='date')
         yield Person(firstname='dummy docker', lastname=date)
         yield Email(address='dummy@email')
-        yield "uno"
+        yield 'uno'
         yield 222

salver/agent/collectors/infrastructure/dirsearch/collector.py

@@ -17,7 +17,5 @@ def from_domain(self, domain):
         data = self.run_container(
             command=['-u', domain.fqdn, '-F', '--timeout=5', '-q', '-t', '4'],
         )
-        print('!!!!', data)
-
         for item in self.findall_regex(data, r'2\d\d - .* - ([^\s]+)'):
             yield Uri(location=item)

salver/agent/services/collectors.py

@@ -24,16 +24,16 @@ def build():
             collectors[instance.config.name] = {
                 'instance': instance,
                 'enabled': instance.config.name in enabled_collectors,
-                'allowed_input': instance.callback_types
+                'allowed_input': instance.callback_types,
             }
         except InvalidCollectorDefinition as err:
-            print(f'Could not load {collector}: {err}')
+            logger.error(f'Could not load {collector}: {err}')
 
     for enabled_collector in enabled_collectors:
         if enabled_collector not in collectors:
             raise MissingCollectorDefinition(enabled_collector)
 
     for collector_name, collector_config in collectors.items():
         enabled = ': enabled' if collector_config['enabled'] else ''
-        logger.info(f'Loaded collector {collector_name}{enabled}')
+        logger.info(f'Loaded collector {collector_name} -> {enabled}')
     return collectors

salver/agent/services/kafka_producers.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
+from salver.common import models
 from salver.config import agent_config
 from salver.common.kafka import Producer
-from salver.common import models
+
 
 def make_error():
     return Producer(
@@ -24,6 +25,7 @@ def make_agent_connect():
         },
     )
 
+
 def make_collect_response():
     return Producer(
         topic='collect-response',
@@ -34,6 +36,7 @@ def make_collect_response():
         },
     )
 
+
 def make_collect_done():
     return Producer(
         topic='collect-done',
@@ -44,6 +47,7 @@ def make_collect_done():
         },
     )
 
+
 def make_agent_disconnect():
     return Producer(
         topic='agent-disconnect',

salver/agent/tasks.py

@@ -14,7 +14,7 @@ def __init__(self, agent_info):
         self.agent_info = agent_info
 
     def on_message(self, engine_info):
-        logger.info(f'Got engine connect: {engine_info}')
+        logger.info(f'Got engine connect: {engine_info.name}')
         self.agent_connect.produce(self.agent_info, flush=True)
 
 
@@ -25,45 +25,56 @@ def __init__(self, enabled_collectors):
         self.collect_finished_p = kafka_producers.make_collect_done()
         self.error_p = kafka_producers.make_error()
 
-    def execute_collector(self, collect):
+    def execute_collector(self, collect: models.Collect):
         start_time = timer()
         collect_output = self.enabled_collectors[collect.collector_name].collect(
-            collect.external_id.hex, collect.facts,
+            collect.external_id.hex,
+            collect.facts,
         )
         state = models.CollectState.FINISHED
         facts_count = 0
         for out in collect_output:
             if isinstance(out, models.BaseFact):
-                self.collect_response_p.produce(models.CollectResponse(
-                    collect_id=collect.external_id,
-                    scan_id=collect.scan_id,
-                    fact=out
-                ))
+                self.collect_response_p.produce(
+                    models.CollectResult(
+                        collect_id=collect.external_id,
+                        scan_id=collect.scan_id,
+                        fact=out,
+                        collector_name=collect.collector_name,
+                    ),
+                )
                 facts_count = facts_count + 1
             else:
                 state = models.CollectState.ERRORED
                 self.error_p.produce(out)
-                print("GOT AN ERROR!!!", out)
-        
+                logger.error(f'OnCollect error: {out.error}')
+
         elapsed_time = timer() - start_time
         return state, elapsed_time, facts_count
 
     def on_message(self, collect):
-        logger.info(f'Got agent collect: {collect}')
+        logger.info(f'Got collect for {collect.collector_name}')
         if collect.collector_name not in self.enabled_collectors:
-            error = f'Collector {collect.collector_name} does not exists (or not enabled).'
+            error = (
+                f'Collector {collect.collector_name} does not exists (or not enabled).'
+            )
             logger.warning(error)
-            return self.error_p.produce(models.Error(
-                context=f"agent-collect.collector_not_found",
-                error=error,
-                collect_id=collect.external_id.hex,
-                collector_name=collect.collector_name
-            ))
+            return self.error_p.produce(
+                models.Error(
+                    context=f'agent-collect.collector_not_found',
+                    error=error,
+                    collect_id=collect.external_id.hex,
+                    collector_name=collect.collector_name,
+                ),
+            )
 
         state, elapsed_time, facts_count = self.execute_collector(collect)
-        self.collect_finished_p.produce(models.CollectDone(
-            collect_id=collect.external_id,
-            state=state,
-            facts_count=facts_count,
-            duration=elapsed_time
-        ))
+        self.collect_finished_p.produce(
+            models.CollectDone(
+                collect_id=collect.external_id,
+                state=state,
+                facts_count=facts_count,
+                duration=elapsed_time,
+                collector_name=collect.collector_name,
+            ),
+        )

salver/common/collectors/base.py

@@ -44,7 +44,7 @@ def __init__(self):
 
     @property
     def callback_types(self):
-        return [ c.schema()["title"] for c in self.callbacks().keys() ]
+        return [c.schema()['title'] for c in self.callbacks().keys()]
 
     def configure(self):
         self.config = CollectorBaseConfig(**self.config)
@@ -65,12 +65,14 @@ def _sanitize_output(self, collect_id, fn):
                 if isinstance(out, models.BaseFact):
                     yield out
                 else:
-                    error = f'Found unknown output from collector {self.config.name}: {out}'
+                    error = (
+                        f'Found unknown output from collector {self.config.name}: {out}'
+                    )
                     yield models.Error(
-                        context=f"agent-collect.unknown_output",
+                        context=f'agent-collect.unknown_output',
                         collect_id=collect_id,
                         error=error,
-                        collector_name=self.config.name
+                        collector_name=self.config.name,
                     )
                     logger.warning(error)
 
@@ -79,10 +81,10 @@ def _sanitize_output(self, collect_id, fn):
                 f'Error while executing callback from {self.config.name}: {type(err).__name__} {err}',
             )
             yield models.Error(
-                context=f"agent-collect.error",
+                context=f'agent-collect.error',
                 error=str(err),
                 collect_id=collect_id,
-                collector_name=self.config.name
+                collector_name=self.config.name,
             )
 
     def _prepare_callbacks(
@@ -99,11 +101,6 @@ def _prepare_callbacks(
     def collect(self, collect_id, facts: List[models.BaseFact]):
         callbacks = self._prepare_callbacks(facts)
 
-        logger.debug(
-            f'Execute collector {self.config.name} with \
-            {len(facts)} facts and {len(callbacks)} callbacks',
-        )
-
         for cb in callbacks:
             yield from self._sanitize_output(collect_id, cb)
 

salver/common/kafka/consumer.py

@@ -18,13 +18,9 @@
 def _process_msg(q, consumer, callback, topic):
     msg = q.get(timeout=60)
     value = msg.value()
-    logger.debug(f'From {topic} got {value}')
-
     callback(value)
-
     q.task_done()
     consumer.commit(msg)
-    logger.debug(f'Task done {topic}')
 
 
 class ConsumerCallback(ABC):

salver/common/kafka/producer.py

@@ -44,7 +44,7 @@ def _delivery_report(err, msg):
         if err is not None:
             logger.error(f'Delivery failed for {msg.key()}: {err}')
             return
-        logger.debug(f'Produced {str(msg.key())} to {msg.topic()}')
+        # logger.debug(f'Produced {str(msg.key())} to {msg.topic()}')
         # msg.key(),
         # msg.topic(),
         # msg.partition(),
@@ -53,7 +53,7 @@ def _delivery_report(err, msg):
     def produce(self, msg, flush=False):
         self.producer.poll(0.0)
         msg_id = uuid4().hex
-        logger.debug(f'Producing {msg_id} to {self.topic}: {msg}')
+        # logger.debug(f'Producing {msg_id} to {self.topic}: {msg}')
         self.producer.produce(
             topic=self.topic,
             key=msg_id,
@@ -64,5 +64,4 @@ def produce(self, msg, flush=False):
             self.flush()
 
     def flush(self):
-        logger.debug(f'Flush producer for {self.topic}')
         self.producer.flush()

salver/common/models/__init__.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 from .fact import BaseFact
 from .ping import PingRequest
+from .scan import Scan, ScanState, ScanConfig
 from .agent import AgentInfo
+from .error import Error
 from .engine import EngineInfo
-from .collect import Collect, CollectState, CollectResponse, CollectDone
+from .collect import Collect, CollectDone, CollectState, CollectResponse
 from .collector import Collector
-from .scan import Scan, ScanConfig, ScanState
-from .error import Error
 
 __all__ = [
     'BaseFact',
@@ -20,6 +20,6 @@
     'ScanConfig',
     'ScanState',
     'Error',
-    'CollectResponse',
-    'CollectDone'
+    'CollectResult',
+    'CollectDone',
 ]