Skip to content

Commit 2570dd0

Browse files
mickeynmickeyn
authored
Merge pull request #74 from metacpan/mickey/test
Tests - setup + basic tests for all scripts
2 parents 8c65525 + 39d05e4 commit 2570dd0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+997532
-235
lines changed

‎.circleci/config.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
---
2+
# Use the latest 2.1 version of CircleCI pipeline process engine. See:
3+
# https://circleci.com/docs/2.0/configuration-reference
4+
version: 2.1
5+
orbs:
6+
codecov: codecov/codecov@4.1.0
7+
# Orchestrate or schedule a set of jobs
8+
workflows:
9+
docker-compose:
10+
jobs:
11+
- build-and-test
12+
jobs:
13+
build-and-test:
14+
machine: true
15+
resource_class: large
16+
steps:
17+
- run:
18+
name: docker-compose version
19+
command: docker-compose --version
20+
- run:
21+
command: |
22+
git clone https://github.com/metacpan/metacpan-docker.git
23+
cd metacpan-docker
24+
name: metacpan-docker checkout
25+
- checkout:
26+
path: metacpan-docker/src/metacpan-ingest
27+
- run:
28+
command: |
29+
pushd metacpan-docker
30+
./bin/metacpan-docker init
31+
name: clone missing repositories
32+
- run:
33+
command: |
34+
pushd metacpan-docker
35+
docker-compose build --build-arg CPM_ARGS='--with-test' ingest_test
36+
name: compose build
37+
- run:
38+
command: |
39+
pushd metacpan-docker
40+
./bin/metacpan-docker init
41+
#docker-compose --verbose up -d ingest_test
42+
docker-compose --profile ingest-test up --build
43+
name: compose up
44+
- run:
45+
command: |
46+
pushd metacpan-docker
47+
docker-compose exec -T ingest_test cpm install -g Devel::Cover
48+
name: install Devel::Cover
49+
# Since we're running docker-compose -d, we don't actually know if
50+
# Elasticsearch is available at the time this build step begins. We
51+
# probably need to wait for it here, so we'll add our own check.
52+
- run:
53+
command: |
54+
pushd metacpan-docker
55+
./src/metacpan-ingest/wait-for-es.sh http://localhost:9200 elasticsearch_test
56+
name: wait for ES
57+
- run:
58+
command: |
59+
pushd metacpan-docker
60+
docker build . -t metacpan/metacpan-ingest && docker compose run ingest env PLACK_ENV=dev HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,^t/|^test-data/|^etc/" prove -lrv --jobs 4 t
61+
# docker-compose exec -T ingest_test env PLACK_ENV=dev HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,^t/|^test-data/|^etc/" prove -lr --jobs 4 t
62+
name: run tests with coverage
63+
# We are relying on environment variables from the host to be available when
64+
# we publish the report, so we publish from the host rather than trying
65+
# to propagate env variables to the container.
66+
- run:
67+
command: |
68+
pushd metacpan-docker
69+
docker-compose exec -T ingest_test cover -report json
70+
name: create coverage report
71+
- codecov/upload:
72+
file: metacpan-docker/src/metacpan-ingest/cover_db/cover.json
73+
- run:
74+
command: |
75+
pushd metacpan-docker
76+
docker-compose logs
77+
docker stats --no-stream
78+
docker ps -a | head
79+
name: docker-compose logs
80+
when: on_fail

‎.github/workflows/build-container.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ jobs:
3838
cache-from: type=gha
3939
cache-to: type=gha,mode=max
4040
- name: Run Perl tests
41-
run: docker run -i ${{ steps.docker-build-test.outputs.imageid }}
41+
run: docker compose build && docker compose run ingest env PLACK_ENV=dev prove -lv t
4242
- name: Docker meta
4343
id: meta
4444
uses: docker/metadata-action@v5

‎Dockerfile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ COPY lib lib
2626
COPY conf conf
2727
COPY *.conf .
2828

29+
# move the following to test only
30+
COPY t t
31+
COPY test_data test_data
32+
2933
ENV PERL5LIB="/app/local/lib/perl5:/app/lib" PATH="/app/local/bin:${PATH}"
3034

3135
USER metacpan

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[![Build container](https://github.com/metacpan/metacpan-ingest/actions/workflows/build-container.yml/badge.svg)](https://github.com/metacpan/metacpan-ingest/actions/workflows/build-container.yml)

‎bin/author.pl‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,18 +51,18 @@
5151
};
5252

5353
# args
54-
my ( $mode, $pauseid, $whois_file );
54+
my ( $findls_file, $pauseid, $whois_file );
5555
GetOptions(
56-
"mode=s" => \$mode,
57-
"pauseid=s" => \$pauseid,
58-
"whois_file=s" => \$whois_file,
56+
"findls_file=s" => \$findls_file,
57+
"pauseid=s" => \$pauseid,
58+
"whois_file=s" => \$whois_file,
5959
);
6060

6161
# setup
62-
my $es = MetaCPAN::ES->new( index => "author", ( $mode ? ( mode => $mode ) : () ) );
62+
my $es = MetaCPAN::ES->new( index => "author" );
6363

6464
log_info {'Reading 00whois'};
65-
my $authors_data = read_00whois( $whois_file );
65+
my $authors_data = read_00whois($whois_file);
6666

6767
if ($pauseid) {
6868
log_info {"Indexing 1 author"};
@@ -199,7 +199,7 @@ ($id)
199199
return undef
200200
unless $dir->is_dir;
201201

202-
my $cpan_file_map = cpan_file_map();
202+
my $cpan_file_map = cpan_file_map($findls_file);
203203
my $author_cpan_files = $cpan_file_map->{$id}
204204
or return undef;
205205

‎bin/backpan.pl‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,18 @@
99
use MetaCPAN::Ingest qw< cpan_file_map >;
1010

1111
# args
12-
my ( $distribution, $files_only, $undo );
12+
my ( $distribution, $files_only, $findls_file, $undo );
1313
GetOptions(
1414
"distribution=s" => \$distribution,
1515
"files_only" => \$files_only,
16+
"findls_file=s" => \$findls_file,
1617
"undo" => \$undo,
1718
);
1819

1920
# setup
20-
my $cpan_file_map = cpan_file_map();
21-
my $es_release = MetaCPAN::ES->new( type => "release" );
22-
my $es_file = MetaCPAN::ES->new( type => "file" );
21+
my $cpan_file_map = cpan_file_map($findls_file);
22+
my $es_release = MetaCPAN::ES->new( index => "release" );
23+
my $es_file = MetaCPAN::ES->new( index => "file" );
2324

2425
my %bulk;
2526
my %release_status;
@@ -29,6 +30,8 @@
2930
update_files();
3031

3132
$_->flush for values %bulk;
33+
$es_release->index_refresh;
34+
$es_file->index_refresh;
3235

3336
log_info {"done"};
3437

@@ -55,7 +58,7 @@ ()
5558
(
5659
$undo
5760
or exists $cpan_file_map->{$author}{$archive}
58-
)
61+
)
5962
? 'cpan'
6063
: 'backpan',
6164
$release->{_id}
@@ -169,6 +172,6 @@ =head1 DESCRIPTION
169172
Sets "backpan" status on all BackPAN releases.
170173
171174
--undo will set distributions' status back as 'cpan'
172-
--file_only will only fix the 'file' type
175+
--file_only will only fix the 'file' index
173176
174177
=cut

‎bin/backup.pl‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,15 @@
2121
my $size = 1000;
2222
my $index = "cpan";
2323

24-
my ( $dry_run, $mode, $purge, $restore );
24+
my ( $dry_run, $purge, $restore );
2525
GetOptions(
2626
"batch_size=i" => \$batch_size,
2727
"dry_run" => \$dry_run,
2828
"index=s" => \$index,
29-
"mode=s" => \$mode,
3029
"purge" => \$purge,
3130
"restore=s" => \$restore,
3231
"size=i" => \$size,
3332
);
34-
# TODO: find a better way
35-
my @es_mode = ( $mode ? mode => $mode : () );
36-
$mode eq 'test' and Log::Log4perl::init('log4perl_test.conf');
3733

3834
# setup
3935
my $home = home();
@@ -76,7 +72,7 @@ ()
7672
# create a new bulk helper for each
7773
my $key = $raw->{_index};
7874

79-
$es_store{$key} ||= MetaCPAN::ES->new( index => $key, @es_mode );
75+
$es_store{$key} ||= MetaCPAN::ES->new( index => $key );
8076
my $es = $es_store{$key};
8177

8278
$bulk_store{$key} ||= $es->bulk( max_count => $batch_size );
@@ -156,15 +152,14 @@ ()
156152
}
157153

158154
sub run_backup {
159-
my $filename = join( '-',
160-
DateTime->now->strftime('%F'),
161-
grep {defined} $index );
155+
my $filename
156+
= join( '-', DateTime->now->strftime('%F'), grep {defined} $index );
162157

163158
my $file = $home->child( qw< var backup >, "$filename.json.gz" );
164159
$file->parent->mkpath unless ( -e $file->parent );
165160
my $fh = IO::Zlib->new( "$file", 'wb4' );
166161

167-
my $es = MetaCPAN::ES->new( index => $index, @es_mode );
162+
my $es = MetaCPAN::ES->new( index => $index );
168163
my $scroll = $es->scroll(
169164
scroll => '1m',
170165
body => {

‎bin/check.pl‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,18 @@
99
use MetaCPAN::Ingest qw< read_02packages_fh >;
1010

1111
# args
12-
my ( $module, $max_errors, $errors_only );
13-
12+
my ( $errors_only, $max_errors, $module );
1413
GetOptions(
15-
"module=s" => \$module,
16-
"max_errors" => \$max_errors,
1714
"errors_only" => \$errors_only,
15+
"max_errors" => \$max_errors,
16+
"module=s" => \$module,
1817
);
1918

2019
# setup
2120
my $error_count = 0;
2221
my $packages_fh = read_02packages_fh();
23-
my $es_file = MetaCPAN::ES->new( type => "file" );
24-
my $es_release = MetaCPAN::ES->new( type => "release" );
22+
my $es_file = MetaCPAN::ES->new( index => "file" );
23+
my $es_release = MetaCPAN::ES->new( index => "release" );
2524

2625
my $modules_start = 0;
2726
while ( my $line = <$packages_fh> ) {

‎bin/checksum.pl‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,14 @@
1111

1212
# args
1313
my $limit = 1000;
14-
my $dry_run;
14+
my ($dry_run);
1515
GetOptions(
16-
"limit=i" => \$limit,
1716
"dry_run" => \$dry_run,
17+
"limit=i" => \$limit,
1818
);
1919

2020
# setup
21-
my $es = MetaCPAN::ES->new( type => "release" );
21+
my $es = MetaCPAN::ES->new( index => "release" );
2222
my $bulk;
2323
$bulk = $es->bulk() unless $dry_run;
2424

@@ -80,6 +80,7 @@
8080
}
8181

8282
$bulk->flush unless $dry_run;
83+
$es->index_refresh;
8384

8485
log_info {'Finished adding checksums'};
8586

‎bin/contributor.pl‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,20 @@
1717
"all" => \$all,
1818
"distribution=s" => \$distribution,
1919
"release=s" => \$release,
20-
2120
);
2221

2322
# setup
2423
my $author_mapping = {};
2524
my $email_mapping = {};
2625

27-
my $es_author = MetaCPAN::ES->new( type => 'author' );
28-
my $es_release = MetaCPAN::ES->new( type => "release" );
29-
my $es_contributor = MetaCPAN::ES->new( type => "contributor" );
26+
my $es_author = MetaCPAN::ES->new( index => 'author' );
27+
my $es_release = MetaCPAN::ES->new( index => "release" );
28+
my $es_contributor = MetaCPAN::ES->new( index => "contributor" );
3029

3130
run();
3231

32+
$es_contributor->index_refresh;
33+
3334
log_info {"done"};
3435

3536
###
@@ -186,7 +187,7 @@ ($release)
186187
$authors = [ grep { $_ ne 'unknown' } @$authors ];
187188

188189
my $author_email = $author_mapping->{$author_name}
189-
//= eval { $es_author->get_source( id => $author_name )->{email}; }
190+
//= eval { $es_author->get_source($author_name)->{email}; }
190191
or return [];
191192
192193
my $author_info = {
@@ -296,6 +297,6 @@ =head1 SYNOPSIS
296297
=head1 DESCRIPTION
297298
298299
Update the list of contributors (CPAN authors only) of all/matching
299-
releases in the 'contributor' type (index).
300+
releases in the 'contributor' index.
300301
301302
=cut

0 commit comments

Comments
 (0)