From fd2005baf4e9efa0a9996e051e1b7fbf8d3a6cf9 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 18 Oct 2018 21:13:51 -0500 Subject: [PATCH] Additional cleanup and simplification The role is further being pruned and made more efficient. Change-Id: I75de7063164959041193526e519682ba80185250 Signed-off-by: Kevin Carter --- osquery/installOSquery.yml | 1 + osquery/roles/fleet/tasks/fleetMigrateDB.yml | 4 + .../roles/fleet/tasks/fleetRegisterAdmin.yml | 11 ++ osquery/roles/osquery/defaults/main.yml | 108 ++++++------ osquery/roles/osquery/handlers/main.yml | 9 +- .../roles/osquery/tasks/osquery_configure.yml | 2 +- .../roles/osquery/templates/osquery.conf.j2 | 154 +++--------------- .../roles/osquery/templates/osquery.flags.j2 | 5 +- osquery/vars/variables.yml | 13 +- 9 files changed, 113 insertions(+), 194 deletions(-) diff --git a/osquery/installOSquery.yml b/osquery/installOSquery.yml index 1a555626..eede075f 100644 --- a/osquery/installOSquery.yml +++ b/osquery/installOSquery.yml @@ -101,6 +101,7 @@ dest: "{{ item }}" state: "absent" delegate_to: localhost + run_once: true with_items: - "/tmp/{{ kolide_fleet_ssl_cert | basename }}" - "/tmp/{{ kolide_fleet_ssl_key | basename }}" diff --git a/osquery/roles/fleet/tasks/fleetMigrateDB.yml b/osquery/roles/fleet/tasks/fleetMigrateDB.yml index 4994f44f..b5c0693c 100644 --- a/osquery/roles/fleet/tasks/fleetMigrateDB.yml +++ b/osquery/roles/fleet/tasks/fleetMigrateDB.yml @@ -17,3 +17,7 @@ command: /usr/local/bin/fleet prepare db --config=/etc/fleet/fleet_config.yml --no-prompt changed_when: false run_once: true + register: _fleetctl_db_task + until: _fleetctl_db_task is success + retries: 3 + delay: 2 diff --git a/osquery/roles/fleet/tasks/fleetRegisterAdmin.yml b/osquery/roles/fleet/tasks/fleetRegisterAdmin.yml index ac871359..b36cae1e 100644 --- a/osquery/roles/fleet/tasks/fleetRegisterAdmin.yml +++ b/osquery/roles/fleet/tasks/fleetRegisterAdmin.yml @@ -16,6 +16,10 @@ - name: set fleetctl default context command: /usr/local/bin/fleetctl config set --address https://127.0.0.1:{{ kolide_fleet_port }} --tls-skip-verify changed_when: false + register: _fleetctl_set_task + until: _fleetctl_set_task is success + retries: 3 + delay: 2 - name: register admin account command: /usr/local/bin/fleetctl setup --email {{ kolide_fleet_admin_email }} --password {{ kolide_fleet_admin_password }} @@ -24,7 +28,14 @@ - fleet_register_admin.rc == 0 failed_when: - fleet_register_admin.rc not in [0, 1] + until: fleet_register_admin is success + retries: 3 + delay: 2 - name: login admin account command: /usr/local/bin/fleetctl login --email {{ kolide_fleet_admin_email }} --password {{ kolide_fleet_admin_password }} changed_when: false + register: _fleetctl_login_task + until: _fleetctl_login_task is success + retries: 3 + delay: 2 diff --git a/osquery/roles/osquery/defaults/main.yml b/osquery/roles/osquery/defaults/main.yml index 38edad0e..e18e8ba0 100644 --- a/osquery/roles/osquery/defaults/main.yml +++ b/osquery/roles/osquery/defaults/main.yml @@ -2,8 +2,6 @@ # Enable or disable the installation of the osquery debug packages. osquery_debug_packages_install: false -osquery_template: 'osquery.conf.j2' - osquery_upload_packs: - osquery-snapshots-pack - osquery-monitoring2-pack @@ -18,51 +16,38 @@ osquery_packs: - "osquery-snapshots-pack" - "osquery-monitoring2-pack" -osquery_config_plugin: 'filesystem' -osquery_logger_plugin: 'filesystem' +# osquery_flags: +# - '--enroll_secret_path=/etc/osquery/osquery_enroll_secret' +# - '--tls_server_certs=/etc/osquery/kolide.crt' +# - '--tls_hostname=acme.kolide.co' +# - '--host_identifier=hostname' +# - '--enroll_tls_endpoint=/api/v1/osquery/enroll' +# - '--config_plugin=tls' +# - '--config_tls_endpoint=/api/v1/osquery/config' +# - '--config_tls_refresh=10' +# - '--disable_distributed=false' +# - '--distributed_plugin=tls' +# - '--distributed_interval=10' +# - '--distributed_tls_max_attempts=3' +# - '--distributed_tls_read_endpoint=/api/v1/osquery/distributed/read' +# - '--distributed_tls_write_endpoint=/api/v1/osquery/distributed/write' +# - '--logger_plugin=tls' +# - '--logger_tls_endpoint=/api/v1/osquery/log' +# - '--logger_tls_period=10' osquery_flags: [] -## if using zentral, kolide or else -## https://github.com/zentralopensource/zentral/blob/f460b10a95d4ea1e515aea3363f55733465d1d9c/zentral/contrib/osquery/deb_script/template.sh -## https://docs.kolide.co/kolide/current/infrastructure/adding-hosts-to-kolide.html -#osquery_enroll_secret: "" - -#osquery_flags: -# - '--enroll_secret_path=/etc/osquery/osquery_enroll_secret' -# - '--tls_server_certs=/etc/osquery/kolide.crt' -# - '--tls_hostname=acme.kolide.co' -# - '--host_identifier=hostname' -# - '--enroll_tls_endpoint=/api/v1/osquery/enroll' -# - '--config_plugin=tls' -# - '--config_tls_endpoint=/api/v1/osquery/config' -# - '--config_tls_refresh=10' -# - '--disable_distributed=false' -# - '--distributed_plugin=tls' -# - '--distributed_interval=10' -# - '--distributed_tls_max_attempts=3' -# - '--distributed_tls_read_endpoint=/api/v1/osquery/distributed/read' -# - '--distributed_tls_write_endpoint=/api/v1/osquery/distributed/write' -# - '--logger_plugin=tls' -# - '--logger_tls_endpoint=/api/v1/osquery/log' -# - '--logger_tls_period=10' - -osquery_fim: true -osquery_fim_interval: 900 +## Take care if using a lot /tmp. can trigger +## 'Expiring events for subscriber: file_events (overflowed limit 1000)' +## => losing many queries results (fim or not) osquery_fim_filepaths: - name: homes_sshdir list: - "/root/.ssh/%%" - "/home/%/.ssh/%%" + - "/home/lib/%/.ssh/%%" - name: etc list: - "/etc/%%" - -## Take care if using a lot /tmp. can trigger -## 'Expiring events for subscriber: file_events (overflowed limit 1000)' -## => losing many queries results (fim or not) -# - name: tmp -# list: -# - "/tmp/%%" - name: bin list: - "/bin/%%" @@ -82,20 +67,43 @@ osquery_fim_excludepaths: list: - /tmp/too_many_events/ -## queries snapshots: 1/week (or 1/month? nok) +## queries snapshots: 1/week ## Max interval 1/w: https://github.com/theopolis/osquery/commit/b76dee8a1fddccb500bc4a058daa1b39083b9dbb osquery_snapshot_interval: 604800 osquery_snapshot_interval2: 604800 -#osquery_snapshot_interval2: 2592000 -## making schedule query faster to have logs faster... -osquery_testing: false -osquery_testing_pause: false -osquery_testing_fim_load: false -osquery_testing_fim_load_bigger: 100M -osquery_testing_fim_load_smaller: 1000 -## profiling? -osquery_profiling: false -## full config might be too long for travis (max 50min) -#osquery_profiling_conf: /etc/osquery/osquery.conf -osquery_profiling_conf: /usr/share/osquery/packs/incident-response.conf +osquery_options: + options: + ## Splay the scheduled interval for queries. + ## This is very helpful to prevent system performance impact when scheduling + ## large numbers of queries that run a smaller or similar intervals. + schedule_splay_percent: 10 + ## Clear events from the osquery backing store after a number of seconds. + events_expiry: 3600 + ## Enable debug or verbose debug output when logging. + verbose: false + ## The number of threads for concurrent query schedule execution. + worker_threads: 2 + ## Enable schedule profiling, this will fill in averages and totals for + ## system/user CPU time and memory for every query in the schedule. + ## Add a query: "select * from osquery_schedule" to record the performances. + enable_monitor: true + logger_snapshot_event_type: true + ## Define a schedule of queries: + schedule: + ## This is a simple example query that outputs basic system information. + system_info: + ## The exact query to run. + query: "SELECT hostname, cpu_brand, physical_memory FROM system_info;" + ## The interval in seconds to run this query, not an exact interval. + interval: 3600 + fim: + query: "select * from file_events;" + removed: false + ## fim query interval + interval: 900 + ## Decorators are normal queries that append data to every query. + decorators: + load: + - "SELECT uuid AS host_uuid FROM system_info;" + - "SELECT user AS username FROM logged_in_users ORDER BY time DESC LIMIT 1;" diff --git a/osquery/roles/osquery/handlers/main.yml b/osquery/roles/osquery/handlers/main.yml index 920a4118..f85e8aad 100644 --- a/osquery/roles/osquery/handlers/main.yml +++ b/osquery/roles/osquery/handlers/main.yml @@ -1,9 +1,16 @@ --- +- name: Enable and stop the service + systemd: + name: osqueryd + enabled: true + state: stopped + listen: restart osquery + - name: Enable and start the service systemd: name: osqueryd enabled: true - state: restarted + state: started listen: restart osquery - name: Enable and start the service diff --git a/osquery/roles/osquery/tasks/osquery_configure.yml b/osquery/roles/osquery/tasks/osquery_configure.yml index 3a917e2b..3c7dadd8 100644 --- a/osquery/roles/osquery/tasks/osquery_configure.yml +++ b/osquery/roles/osquery/tasks/osquery_configure.yml @@ -26,7 +26,7 @@ - name: Configure osquery template: - src: "{{ osquery_template }}" + src: "osquery.conf.j2" dest: /etc/osquery/osquery.conf mode: '0644' backup: yes diff --git a/osquery/roles/osquery/templates/osquery.conf.j2 b/osquery/roles/osquery/templates/osquery.conf.j2 index 84458524..d8c93f9f 100644 --- a/osquery/roles/osquery/templates/osquery.conf.j2 +++ b/osquery/roles/osquery/templates/osquery.conf.j2 @@ -1,132 +1,30 @@ {{ ansible_managed | comment('c')}} -{ - // Configure the daemon below: - "options": { - // Select the osquery config plugin. - "config_plugin": "{{ osquery_config_plugin }}", +{# Set the FIM include path items #} +{%- if (osquery_fim_filepaths | length) > 0 %} +{%- set file_path_dict = {} %} +{%- for item in osquery_fim_filepaths %} +{%- set _ = file_path_dict.__setitem__(item.name, item.list) -%} +{%- endfor %} +{%- set _ = osquery_options.__setitem__("file_paths", file_path_dict) -%} +{%- endif -%} - // Select the osquery logging plugin. - "logger_plugin": "{{ osquery_logger_plugin }}", +{# Set the FIM exlude path items #} +{%- if (osquery_fim_excludepaths | length) > 0 %} +{%- set exclude_file_path_dict = {} %} +{%- for item in osquery_fim_excludepaths %} +{%- set _ = exclude_file_path_dict.__setitem__(item.name, item.list) -%} +{%- endfor %} +{%- set _ = osquery_options.__setitem__("exclude_paths", exclude_file_path_dict) -%} +{%- endif -%} - // The log directory stores info, warning, and errors. - // If the daemon uses the 'filesystem' logging retriever then the log_dir - // will also contain the query results. - //"logger_path": "/var/log/osquery", +{# Set the loaded packs #} +{%- if (osquery_packs | length) > 0 %} +{%- set pack_dict = {} %} +{%- for item in osquery_packs %} +{%- set _ = pack_dict.__setitem__(item, "/usr/share/osquery/packs/" + item + ".conf") -%} +{%- endfor %} +{%- set _ = osquery_options.__setitem__("packs", pack_dict) -%} +{%- endif -%} - // Set 'disable_logging' to true to prevent writing any info, warning, error - // logs. If a logging plugin is selected it will still write query results. - //"disable_logging": "false", - - // Query differential results are logged as change-events to assist log - // aggregation operations like searching and transactons. - // Set 'log_results_events' to log differentials as transactions. - //"log_result_events": "true", - - // Splay the scheduled interval for queries. - // This is very helpful to prevent system performance impact when scheduling - // large numbers of queries that run a smaller or similar intervals. - "schedule_splay_percent": "10", - - // Write the pid of the osqueryd process to a pidfile/mutex. - //"pidfile": "/var/osquery/osquery.pidfile", - - // Clear events from the osquery backing store after a number of seconds. - "events_expiry": "3600", - - // A filesystem path for disk-based backing storage used for events and - // query results differentials. See also 'use_in_memory_database'. - //"database_path": "/var/osquery/osquery.db", - - // Comma-delimited list of table names to be disabled. - // This allows osquery to be launched without certain tables. - //"disable_tables": "foo_bar,time", - - // Enable debug or verbose debug output when logging. - "verbose": "false", - - // The number of threads for concurrent query schedule execution. - "worker_threads": "2", - - // Enable schedule profiling, this will fill in averages and totals for - // system/user CPU time and memory for every query in the schedule. - // Add a query: "select * from osquery_schedule" to record the performances. - "enable_monitor": "true", - - "logger_snapshot_event_type": "true" - }, - - // Define a schedule of queries: - "schedule": { - // This is a simple example query that outputs basic system information. - "system_info": { - // The exact query to run. - "query": "SELECT hostname, cpu_brand, physical_memory FROM system_info;", - // The interval in seconds to run this query, not an exact interval. -{% if osquery_testing or osquery_testing_pause %} - "interval": 10 -{% else %} - "interval": 3600 -{% endif %} - }{% if osquery_fim %},{% endif %} - -{% if osquery_fim %} - "fim" : { -// "query": "select target_path, category, time, action from file_events;", - "query": "select * from file_events;", - "removed": false, - "interval": {% if osquery_testing_fim_load %}30{% else %}{{ osquery_fim_interval }}{% endif %} - } - -{% endif %} - }, - -{% if osquery_fim %} - "file_paths": { -{% for item in osquery_fim_filepaths %} - "{{ item.name }}": [ -{% for entry in item.list %} - "{{ entry }}"{% if not loop.last %},{% endif %} - -{% endfor %} - ]{% if not loop.last %},{% endif %} - -{% endfor %} - }, - "exclude_paths": { -{% for item in osquery_fim_excludepaths %} - "{{ item.name }}": [ -{% for entry in item.list %} - "{{ entry }}"{% if not loop.last %},{% endif %} - -{% endfor %} - ]{% if not loop.last %},{% endif %} - -{% endfor %} - }, - -{% endif %} - // Decorators are normal queries that append data to every query. - "decorators": { - "load": [ - "SELECT uuid AS host_uuid FROM system_info;", - "SELECT user AS username FROM logged_in_users ORDER BY time DESC LIMIT 1;" - ] - }, - - // Add default osquery packs or install your own. - // - // There are several 'default' packs installed with 'make install' or via - // packages and/or Homebrew. - // - // Linux: /usr/share/osquery/packs - // OS X: /var/osquery/packs - // Homebrew: /usr/local/share/osquery/packs - // make install: {PREFIX}/share/osquery/packs - // - "packs": { -{% for item in osquery_packs %} - "{{ item }}": "/usr/share/osquery/packs/{{ item }}.conf"{% if not loop.last %},{% endif %} - -{% endfor %} - } -} +{# Render the osquery configuration file as json #} +{{ osquery_options | to_nice_json(indent=4) }} diff --git a/osquery/roles/osquery/templates/osquery.flags.j2 b/osquery/roles/osquery/templates/osquery.flags.j2 index 079f872b..30eb8558 100644 --- a/osquery/roles/osquery/templates/osquery.flags.j2 +++ b/osquery/roles/osquery/templates/osquery.flags.j2 @@ -1,6 +1,3 @@ ---pidfile=/var/run/osqueryd.pid {% for flag in osquery_flags %} {{ flag }} -{% endfor %} ---logger_path=/var/log/osquery ---logger_plugin=filesystem \ No newline at end of file +{% endfor %} \ No newline at end of file diff --git a/osquery/vars/variables.yml b/osquery/vars/variables.yml index e190cf27..188d52a1 100644 --- a/osquery/vars/variables.yml +++ b/osquery/vars/variables.yml @@ -1,19 +1,10 @@ --- - -kolide_fleet_enable: true - kolide_fleet_ssl_cert: /etc/ssl/certs/fleet.cert kolide_fleet_ssl_key: /etc/ssl/private/fleet.key kolide_fleet_ssl_ca_cert: /etc/ssl/certs/fleet-ca.pem -kolide_fleet_ssl_ca_key: /etc/ssl/private/fleet-ca.key kolide_fleet_ssl_pem: /etc/ssl/private/fleet.pem -# Osquery vars osquery_enroll_secret_file: /etc/osquery/osquery_enroll_secret -osquery_debug_packages_install: false -osquery_config_plugin: 'filesystem' -osquery_logger_plugin: 'filesystem' - osquery_flags: - "--tls_server_certs={{ kolide_fleet_ssl_cert }}" - "--tls_hostname={{ hostvars[groups['kolide-fleet_all'][0]]['ansible_host'] }}:443" @@ -28,7 +19,9 @@ osquery_flags: - "--distributed_tls_max_attempts=3" - "--distributed_tls_read_endpoint=/api/v1/osquery/distributed/read" - "--distributed_tls_write_endpoint=/api/v1/osquery/distributed/write" - - "--logger_plugin=tls" + - "--logger_path=/var/log/osquery" + - "--logger_plugin=filesystem,tls" - "--logger_tls_endpoint=/api/v1/osquery/log" - "--logger_tls_period=10" - "--enroll_secret_path={{ osquery_enroll_secret_file }}" + - "--pidfile=/var/run/osqueryd.pid"