diff --git a/.dockerignore b/.dockerignore index 706d0164..9e2ccb20 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,5 +5,19 @@ .travis.yml Dockerfile spec -#IDEs folders -.idea \ No newline at end of file + +# Ignore editor specific configs +/.idea +/.vscode +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace +.generators +.rakeTasks + +# System Files +.DS_Store +Thumbs.db diff --git a/.gitignore b/.gitignore index f6d7fbd9..75650bd8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,19 @@ play .jdk-overlay .*env coverage/ + +# Ignore editor specific configs +/.idea +/.vscode +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace +.generators +.rakeTasks + +# System Files +.DS_Store +Thumbs.db diff --git a/Gemfile.lock b/Gemfile.lock index 996fc318..9c8701c8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -752,7 +752,7 @@ GEM multi_json (~> 1.0) pusher-signature (~> 0.1.8) pusher-signature (0.1.8) - rack (2.0.8) + rack (2.2.6.4) rack-protection (2.0.4) rack rack-ssl (1.4.1) diff --git a/db/deploy/create_scan_results_table.sql b/db/deploy/create_scan_results_table.sql new file mode 100644 index 00000000..279741bb --- /dev/null +++ b/db/deploy/create_scan_results_table.sql @@ -0,0 +1,41 @@ +-- Deploy travis-logs:create_scan_results_table to pg + +BEGIN; + + SET client_min_messages = WARNING; + + CREATE TABLE scan_results ( + id bigint NOT NULL, + repository_id bigint NOT NULL, + job_id bigint NOT NULL, + log_id bigint NOT NULL, + owner_id integer NOT NULL, + owner_type character varying NOT NULL, + content jsonb NOT NULL, + issues_found integer NOT NULL, + archived boolean, + purged_at timestamp without time zone, + created_at timestamp without time zone + ); + + CREATE SEQUENCE scan_results_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + ALTER SEQUENCE scan_results_id_seq OWNED BY scan_results.id; + + ALTER TABLE ONLY scan_results + ALTER COLUMN id + SET DEFAULT nextval('scan_results_id_seq'::regclass); + + ALTER TABLE ONLY scan_results + ADD CONSTRAINT scan_results_pkey PRIMARY KEY (id); + + CREATE INDEX index_scan_results_on_repository_id + ON scan_results + USING btree (repository_id); + +COMMIT; diff --git a/db/deploy/create_scan_tracker_table.sql b/db/deploy/create_scan_tracker_table.sql new file mode 100644 index 00000000..1c6a3e08 --- /dev/null +++ b/db/deploy/create_scan_tracker_table.sql @@ -0,0 +1,32 @@ +-- Deploy travis-logs:create_scan_tracker_table to pg +-- requires: logs_create_scan_status + +BEGIN; + + SET client_min_messages = WARNING; + + CREATE TABLE scan_tracker ( + id bigint NOT NULL, + log_id bigint NOT NULL, + scan_status character varying, + details jsonb, + created_at timestamp without time zone + ); + + CREATE SEQUENCE scan_tracker_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + ALTER SEQUENCE scan_tracker_id_seq OWNED BY scan_tracker.id; + + ALTER TABLE ONLY scan_tracker + ALTER COLUMN id + SET DEFAULT nextval('scan_tracker_id_seq'::regclass); + + ALTER TABLE ONLY scan_tracker + ADD CONSTRAINT scan_tracker_pkey PRIMARY KEY (id); + +COMMIT; diff --git a/db/deploy/logs_create_scan_status.sql b/db/deploy/logs_create_scan_status.sql new file mode 100644 index 00000000..4702e348 --- /dev/null +++ b/db/deploy/logs_create_scan_status.sql @@ -0,0 +1,27 @@ +-- Deploy travis-logs:logs_create_scan_status to pg +-- requires: partman_remove_constraint + +BEGIN; + + SET client_min_messages = WARNING; + + ALTER TABLE logs + ADD COLUMN scan_status character varying, + ADD COLUMN scan_status_updated_at timestamp without time zone, + ADD COLUMN censored boolean, + ADD COLUMN scan_queued_at timestamp without time zone, + ADD COLUMN scan_started_at timestamp without time zone, + ADD COLUMN scan_processing_at timestamp without time zone, + ADD COLUMN scan_finalizing_at timestamp without time zone, + ADD COLUMN scan_ended_at timestamp without time zone; + + CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_order_by_newest ON public.logs USING btree (scan_status, id DESC); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_and_scan_status_updated_at ON public.logs USING btree (scan_status, scan_status_updated_at); + -- CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_and_scan_status_updated_at_where_running ON public.logs USING btree (scan_status, scan_status_updated_at) WHERE ((scan_status)::text = ANY ((ARRAY['started'::character varying, 'processing'::character varying, 'finalizing'::character varying])::text[])); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_queued_at ON public.logs USING btree (scan_queued_at); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_started_at ON public.logs USING btree (scan_started_at); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_processing_at ON public.logs USING btree (scan_processing_at); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_finalizing_at ON public.logs USING btree (scan_finalizing_at); + CREATE INDEX IF NOT EXISTS index_logs_on_scan_ended_at ON public.logs USING btree (scan_ended_at); + +COMMIT; diff --git a/db/revert/create_scan_results_table.sql b/db/revert/create_scan_results_table.sql new file mode 100644 index 00000000..b53ed381 --- /dev/null +++ b/db/revert/create_scan_results_table.sql @@ -0,0 +1,9 @@ +-- Revert travis-logs:create_scan_results_table from pg + +BEGIN; + + SET client_min_messages = WARNING; + + DROP TABLE scan_results; + +COMMIT; diff --git a/db/revert/create_scan_tracker_table.sql b/db/revert/create_scan_tracker_table.sql new file mode 100644 index 00000000..d197de25 --- /dev/null +++ b/db/revert/create_scan_tracker_table.sql @@ -0,0 +1,9 @@ +-- Revert travis-logs:create_scan_tracker_table from pg + +BEGIN; + + SET client_min_messages = WARNING; + + DROP TABLE scan_tracker CASCADE; + +COMMIT; diff --git a/db/revert/logs_create_scan_status.sql b/db/revert/logs_create_scan_status.sql new file mode 100644 index 00000000..d8d2acfd --- /dev/null +++ b/db/revert/logs_create_scan_status.sql @@ -0,0 +1,25 @@ +-- Revert travis-logs:logs_create_scan_status from pg + +BEGIN; + + SET client_min_messages = WARNING; + + ALTER TABLE logs + DROP COLUMN scan_status, + DROP COLUMN scan_status_updated_at, + DROP COLUMN censored, + DROP COLUMN scan_queued_at, + DROP COLUMN scan_started_at, + DROP COLUMN scan_processing_at, + DROP COLUMN scan_finalizing_at, + DROP COLUMN scan_ended_at; + + DROP INDEX index_logs_on_scan_status_order_by_newest; + DROP INDEX index_logs_on_scan_status_and_scan_status_updated_at; + DROP INDEX index_logs_on_scan_queued_at; + DROP INDEX index_logs_on_scan_started_at; + DROP INDEX index_logs_on_scan_processing_at; + DROP INDEX index_logs_on_scan_finalizing_at; + DROP INDEX index_logs_on_scan_ended_at; + +COMMIT; diff --git a/db/sqitch.plan b/db/sqitch.plan index 7578ff8a..c074c0de 100644 --- a/db/sqitch.plan +++ b/db/sqitch.plan @@ -6,3 +6,6 @@ vacuum_settings [structure] 2017-04-04T19:37:24Z Dan Buch # log_parts_created_at_not_null [structure] 2017-04-04T19:52:23Z Dan Buch # Modify log_parts.created_at to be NOT NULL with default for use with partman partman [log_parts_created_at_not_null] 2017-04-04T20:24:49Z Dan Buch # Enable and configure partman for log_parts partman_remove_constraint 2018-04-27T11:41:39Z Igor Wiedler # Remove partman constraint exclusion on log_id column +logs_create_scan_status 2022-08-05T12:21:22Z Andrii Mysko # Add scan status columns to logs table +create_scan_tracker_table 2022-08-05T12:21:23Z Andrii Mysko # Add scan_tracker table +create_scan_results_table 2022-09-05T14:31:43Z Stanislav Colotinschi # Add scan_results table diff --git a/db/verify/create_scan_results_table.sql b/db/verify/create_scan_results_table.sql new file mode 100644 index 00000000..53cdbabb --- /dev/null +++ b/db/verify/create_scan_results_table.sql @@ -0,0 +1,11 @@ +-- Verify travis-logs:create_scan_results_table on pg + +BEGIN; + + SET client_min_messages = WARNING; + + SELECT id + FROM scan_results + WHERE false; + +ROLLBACK; diff --git a/db/verify/create_scan_tracker_table.sql b/db/verify/create_scan_tracker_table.sql new file mode 100644 index 00000000..c9efa564 --- /dev/null +++ b/db/verify/create_scan_tracker_table.sql @@ -0,0 +1,11 @@ +-- Verify travis-logs:create_scan_tracker_table on pg + +BEGIN; + + SET client_min_messages = WARNING; + + SELECT id, scan_status, details, created_at + FROM scan_tracker + WHERE false; + +COMMIT; diff --git a/db/verify/logs_create_scan_status.sql b/db/verify/logs_create_scan_status.sql new file mode 100644 index 00000000..6e34eb42 --- /dev/null +++ b/db/verify/logs_create_scan_status.sql @@ -0,0 +1,11 @@ +-- Verify travis-logs:logs_create_scan_status on pg + +BEGIN; + + SET client_min_messages = WARNING; + + SELECT scan_status, scan_status_updated_at, censored, scan_queued_at, scan_started_at, scan_processing_at, scan_finalizing_at, scan_ended_at + FROM logs + WHERE false; + +COMMIT; diff --git a/lib/travis/logs/database.rb b/lib/travis/logs/database.rb index 04c67fb1..c763bfd0 100644 --- a/lib/travis/logs/database.rb +++ b/lib/travis/logs/database.rb @@ -149,6 +149,24 @@ def create_log(job_id) db[:logs].insert(job_id: job_id, created_at: now, updated_at: now) end + def create_scan_tracker_entry(log_id, scan_status) + maint.restrict! + db[:scan_tracker].insert({ + log_id: log_id, + scan_status: scan_status, + created_at: Time.now.utc + }) + end + + def update_log_scan_status(log_id, scan_status) + db.transaction do + db[:logs] + .where(id: log_id) + .update(scan_status_updated_at: Time.now.utc, scan_status: scan_status) + create_scan_tracker_entry(log_id, scan_status) + end + end + def create_log_part(params) maint.restrict! db[:log_parts].insert(params.merge(created_at: Time.now.utc)) diff --git a/lib/travis/logs/s3.rb b/lib/travis/logs/s3.rb index 08e11249..ba4c1c9c 100644 --- a/lib/travis/logs/s3.rb +++ b/lib/travis/logs/s3.rb @@ -10,7 +10,7 @@ module Logs class S3 def self.setup Aws.config.update( - region: 'us-east-1', + region: ENV['TRAVIS_LOGS_S3_REGION'] || 'us-east-1', credentials: Aws::Credentials.new( Travis.config.s3.access_key_id, Travis.config.s3.secret_access_key diff --git a/lib/travis/logs/services/aggregate_logs.rb b/lib/travis/logs/services/aggregate_logs.rb index 694edcd0..74b33b76 100644 --- a/lib/travis/logs/services/aggregate_logs.rb +++ b/lib/travis/logs/services/aggregate_logs.rb @@ -96,6 +96,7 @@ def aggregate_log(log_id) measure do database.db.transaction do aggregate(log_id) + database.update_log_scan_status(log_id, 'ready_for_scan') clean(log_id) unless skip_empty? && log_empty?(log_id) end end