diff --git a/.dockerignore b/.dockerignore index 706d0164..9e2ccb20 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,5 +5,19 @@ .travis.yml Dockerfile spec -#IDEs folders -.idea \ No newline at end of file + +# Ignore editor specific configs +/.idea +/.vscode +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace +.generators +.rakeTasks + +# System Files +.DS_Store +Thumbs.db diff --git a/.gitignore b/.gitignore index f6d7fbd9..75650bd8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,19 @@ play .jdk-overlay .*env coverage/ + +# Ignore editor specific configs +/.idea +/.vscode +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace +.generators +.rakeTasks + +# System Files +.DS_Store +Thumbs.db diff --git a/Gemfile.lock b/Gemfile.lock index 996fc318..a1d30562 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -656,7 +656,6 @@ GEM aws-sigv4 (~> 1.0) aws-sigv2 (1.0.1) aws-sigv4 (1.0.3) - backports (3.11.4) bunny (2.12.0) amq-protocol (~> 2.3, >= 2.3.0) coder (0.4.0) @@ -714,7 +713,7 @@ GEM http-form_data (2.1.1) http_parser.rb (0.6.0) httpclient (2.8.3) - i18n (1.8.2) + i18n (1.12.0) concurrent-ruby (~> 1.0) jaro_winkler (1.5.1) jmespath (1.4.0) @@ -725,11 +724,12 @@ GEM http (>= 2.0, < 4.0) memoist (0.16.0) method_source (0.9.0) - minitest (5.14.1) + minitest (5.18.0) msgpack (1.2.4) multi_json (1.13.1) multipart-post (2.0.0) - mustermann (1.0.3) + mustermann (3.0.0) + ruby2_keywords (~> 0.0.1) opencensus (0.4.0) opencensus-stackdriver (0.2.0) concurrent-ruby (~> 1.0) @@ -752,8 +752,8 @@ GEM multi_json (~> 1.0) pusher-signature (~> 0.1.8) pusher-signature (0.1.8) - rack (2.0.8) - rack-protection (2.0.4) + rack (2.2.6.4) + rack-protection (3.0.6) rack rack-ssl (1.4.1) rack @@ -765,7 +765,7 @@ GEM ffi (>= 1.0.6) msgpack (>= 0.4.3) optimist (>= 3.0.0) - redis (4.0.2) + redis (4.8.1) redis-namespace (1.6.0) redis (>= 3.0.4) redlock (0.2.2) @@ -793,13 +793,14 @@ GEM ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) ruby-progressbar (1.10.0) + ruby2_keywords (0.0.5) sentry-raven (2.7.4) faraday (>= 0.7.6, < 1.0) sequel (5.13.0) - sidekiq (5.2.2) - connection_pool (~> 2.2, >= 2.2.2) - rack-protection (>= 1.5.0) - redis (>= 3.3.5, < 5) + sidekiq (6.5.5) + connection_pool (>= 2.2.2) + rack (~> 2.0) + redis (>= 4.5.0) signet (0.11.0) addressable (~> 2.3) faraday (~> 0.9) @@ -810,29 +811,27 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) - sinatra (2.0.4) - mustermann (~> 1.0) - rack (~> 2.0) - rack-protection (= 2.0.4) + sinatra (3.0.6) + mustermann (~> 3.0) + rack (~> 2.2, >= 2.2.4) + rack-protection (= 3.0.6) tilt (~> 2.0) - sinatra-contrib (2.0.4) - activesupport (>= 4.0.0) - backports (>= 2.8.2) + sinatra-contrib (3.0.6) multi_json - mustermann (~> 1.0) - rack-protection (= 2.0.4) - sinatra (= 2.0.4) - tilt (>= 1.3, < 3) + mustermann (~> 3.0) + rack-protection (= 3.0.6) + sinatra (= 3.0.6) + tilt (~> 2.0) sinatra-param (1.5.0) sinatra (>= 1.3) stackdriver-core (1.3.3) google-cloud-core (~> 1.2) stackprof (0.2.12) thread_safe (0.3.6) - tilt (2.0.8) + tilt (2.1.0) travis-config (1.1.3) hashr (~> 2.0) - tzinfo (1.2.7) + tzinfo (1.2.11) thread_safe (~> 0.1) unf (0.1.4) unf_ext diff --git a/db/deploy/create_scan_results_table.sql b/db/deploy/create_scan_results_table.sql new file mode 100644 index 00000000..279741bb --- /dev/null +++ b/db/deploy/create_scan_results_table.sql @@ -0,0 +1,41 @@ +-- Deploy travis-logs:create_scan_results_table to pg + +BEGIN; + + SET client_min_messages = WARNING; + + CREATE TABLE scan_results ( + id bigint NOT NULL, + repository_id bigint NOT NULL, + job_id bigint NOT NULL, + log_id bigint NOT NULL, + owner_id integer NOT NULL, + owner_type character varying NOT NULL, + content jsonb NOT NULL, + issues_found integer NOT NULL, + archived boolean, + purged_at timestamp without time zone, + created_at timestamp without time zone + ); + + CREATE SEQUENCE scan_results_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + ALTER SEQUENCE scan_results_id_seq OWNED BY scan_results.id; + + ALTER TABLE ONLY scan_results + ALTER COLUMN id + SET DEFAULT nextval('scan_results_id_seq'::regclass); + + ALTER TABLE ONLY scan_results + ADD CONSTRAINT scan_results_pkey PRIMARY KEY (id); + + CREATE INDEX index_scan_results_on_repository_id + ON scan_results + USING btree (repository_id); + +COMMIT; diff --git a/db/deploy/create_scan_tracker_table.sql b/db/deploy/create_scan_tracker_table.sql new file mode 100644 index 00000000..1c6a3e08 --- /dev/null +++ b/db/deploy/create_scan_tracker_table.sql @@ -0,0 +1,32 @@ +-- Deploy travis-logs:create_scan_tracker_table to pg +-- requires: logs_create_scan_status + +BEGIN; + + SET client_min_messages = WARNING; + + CREATE TABLE scan_tracker ( + id bigint NOT NULL, + log_id bigint NOT NULL, + scan_status character varying, + details jsonb, + created_at timestamp without time zone + ); + + CREATE SEQUENCE scan_tracker_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + ALTER SEQUENCE scan_tracker_id_seq OWNED BY scan_tracker.id; + + ALTER TABLE ONLY scan_tracker + ALTER COLUMN id + SET DEFAULT nextval('scan_tracker_id_seq'::regclass); + + ALTER TABLE ONLY scan_tracker + ADD CONSTRAINT scan_tracker_pkey PRIMARY KEY (id); + +COMMIT; diff --git a/db/deploy/logs_create_scan_status.sql b/db/deploy/logs_create_scan_status.sql new file mode 100644 index 00000000..4702e348 --- /dev/null +++ b/db/deploy/logs_create_scan_status.sql @@ -0,0 +1,27 @@ +-- Deploy travis-logs:logs_create_scan_status to pg +-- requires: partman_remove_constraint + +BEGIN; + + SET client_min_messages = WARNING; + + ALTER TABLE logs + ADD COLUMN scan_status character varying, + ADD COLUMN scan_status_updated_at timestamp without time zone, + ADD COLUMN censored boolean, + ADD COLUMN scan_queued_at timestamp without time zone, + ADD COLUMN scan_started_at timestamp without time zone, + ADD COLUMN scan_processing_at timestamp without time zone, + ADD COLUMN scan_finalizing_at timestamp without time zone, + ADD COLUMN scan_ended_at timestamp without time zone; + + CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_order_by_newest ON public.logs USING btree (scan_status, id DESC); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_and_scan_status_updated_at ON public.logs USING btree (scan_status, scan_status_updated_at); + -- CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_and_scan_status_updated_at_where_running ON public.logs USING btree (scan_status, scan_status_updated_at) WHERE ((scan_status)::text = ANY ((ARRAY['started'::character varying, 'processing'::character varying, 'finalizing'::character varying])::text[])); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_queued_at ON public.logs USING btree (scan_queued_at); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_started_at ON public.logs USING btree (scan_started_at); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_processing_at ON public.logs USING btree (scan_processing_at); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_finalizing_at ON public.logs USING btree (scan_finalizing_at); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_ended_at ON public.logs USING btree (scan_ended_at); + +COMMIT; diff --git a/db/revert/create_scan_results_table.sql b/db/revert/create_scan_results_table.sql new file mode 100644 index 00000000..b53ed381 --- /dev/null +++ b/db/revert/create_scan_results_table.sql @@ -0,0 +1,9 @@ +-- Revert travis-logs:create_scan_results_table from pg + +BEGIN; + + SET client_min_messages = WARNING; + + DROP TABLE scan_results; + +COMMIT; diff --git a/db/revert/create_scan_tracker_table.sql b/db/revert/create_scan_tracker_table.sql new file mode 100644 index 00000000..d197de25 --- /dev/null +++ b/db/revert/create_scan_tracker_table.sql @@ -0,0 +1,9 @@ +-- Revert travis-logs:create_scan_tracker_table from pg + +BEGIN; + + SET client_min_messages = WARNING; + + DROP TABLE scan_tracker CASCADE; + +COMMIT; diff --git a/db/revert/logs_create_scan_status.sql b/db/revert/logs_create_scan_status.sql new file mode 100644 index 00000000..d8d2acfd --- /dev/null +++ b/db/revert/logs_create_scan_status.sql @@ -0,0 +1,25 @@ +-- Revert travis-logs:logs_create_scan_status from pg + +BEGIN; + + SET client_min_messages = WARNING; + + ALTER TABLE logs + DROP COLUMN scan_status, + DROP COLUMN scan_status_updated_at, + DROP COLUMN censored, + DROP COLUMN scan_queued_at, + DROP COLUMN scan_started_at, + DROP COLUMN scan_processing_at, + DROP COLUMN scan_finalizing_at, + DROP COLUMN scan_ended_at; + + DROP INDEX index_logs_on_scan_status_order_by_newest; + DROP INDEX index_logs_on_scan_status_and_scan_status_updated_at; + DROP INDEX index_logs_on_scan_queued_at; + DROP INDEX index_logs_on_scan_started_at; + DROP INDEX index_logs_on_scan_processing_at; + DROP INDEX index_logs_on_scan_finalizing_at; + DROP INDEX index_logs_on_scan_ended_at; + +COMMIT; diff --git a/db/sqitch.plan b/db/sqitch.plan index 7578ff8a..c074c0de 100644 --- a/db/sqitch.plan +++ b/db/sqitch.plan @@ -6,3 +6,6 @@ vacuum_settings [structure] 2017-04-04T19:37:24Z Dan Buch # log_parts_created_at_not_null [structure] 2017-04-04T19:52:23Z Dan Buch # Modify log_parts.created_at to be NOT NULL with default for use with partman partman [log_parts_created_at_not_null] 2017-04-04T20:24:49Z Dan Buch # Enable and configure partman for log_parts partman_remove_constraint 2018-04-27T11:41:39Z Igor Wiedler # Remove partman constraint exclusion on log_id column +logs_create_scan_status 2022-08-05T12:21:22Z Andrii Mysko # Add scan status columns to logs table +create_scan_tracker_table 2022-08-05T12:21:23Z Andrii Mysko # Add scan_tracker table +create_scan_results_table 2022-09-05T14:31:43Z Stanislav Colotinschi # Add scan_results table diff --git a/db/verify/create_scan_results_table.sql b/db/verify/create_scan_results_table.sql new file mode 100644 index 00000000..53cdbabb --- /dev/null +++ b/db/verify/create_scan_results_table.sql @@ -0,0 +1,11 @@ +-- Verify travis-logs:create_scan_results_table on pg + +BEGIN; + + SET client_min_messages = WARNING; + + SELECT id + FROM scan_results + WHERE false; + +ROLLBACK; diff --git a/db/verify/create_scan_tracker_table.sql b/db/verify/create_scan_tracker_table.sql new file mode 100644 index 00000000..c9efa564 --- /dev/null +++ b/db/verify/create_scan_tracker_table.sql @@ -0,0 +1,11 @@ +-- Verify travis-logs:create_scan_tracker_table on pg + +BEGIN; + + SET client_min_messages = WARNING; + + SELECT id, scan_status, details, created_at + FROM scan_tracker + WHERE false; + +COMMIT; diff --git a/db/verify/logs_create_scan_status.sql b/db/verify/logs_create_scan_status.sql new file mode 100644 index 00000000..6e34eb42 --- /dev/null +++ b/db/verify/logs_create_scan_status.sql @@ -0,0 +1,11 @@ +-- Verify travis-logs:logs_create_scan_status on pg + +BEGIN; + + SET client_min_messages = WARNING; + + SELECT scan_status, scan_status_updated_at, censored, scan_queued_at, scan_started_at, scan_processing_at, scan_finalizing_at, scan_ended_at + FROM logs + WHERE false; + +COMMIT; diff --git a/lib/travis/logs/database.rb b/lib/travis/logs/database.rb index 04c67fb1..c763bfd0 100644 --- a/lib/travis/logs/database.rb +++ b/lib/travis/logs/database.rb @@ -149,6 +149,24 @@ def create_log(job_id) db[:logs].insert(job_id: job_id, created_at: now, updated_at: now) end + def create_scan_tracker_entry(log_id, scan_status) + maint.restrict! + db[:scan_tracker].insert({ + log_id: log_id, + scan_status: scan_status, + created_at: Time.now.utc + }) + end + + def update_log_scan_status(log_id, scan_status) + db.transaction do + db[:logs] + .where(id: log_id) + .update(scan_status_updated_at: Time.now.utc, scan_status: scan_status) + create_scan_tracker_entry(log_id, scan_status) + end + end + def create_log_part(params) maint.restrict! db[:log_parts].insert(params.merge(created_at: Time.now.utc)) diff --git a/lib/travis/logs/s3.rb b/lib/travis/logs/s3.rb index 08e11249..ba4c1c9c 100644 --- a/lib/travis/logs/s3.rb +++ b/lib/travis/logs/s3.rb @@ -10,7 +10,7 @@ module Logs class S3 def self.setup Aws.config.update( - region: 'us-east-1', + region: ENV['TRAVIS_LOGS_S3_REGION'] || 'us-east-1', credentials: Aws::Credentials.new( Travis.config.s3.access_key_id, Travis.config.s3.secret_access_key diff --git a/lib/travis/logs/services/aggregate_logs.rb b/lib/travis/logs/services/aggregate_logs.rb index 694edcd0..74b33b76 100644 --- a/lib/travis/logs/services/aggregate_logs.rb +++ b/lib/travis/logs/services/aggregate_logs.rb @@ -96,6 +96,7 @@ def aggregate_log(log_id) measure do database.db.transaction do aggregate(log_id) + database.update_log_scan_status(log_id, 'ready_for_scan') clean(log_id) unless skip_empty? && log_empty?(log_id) end end