diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c5f27e4..0f57e65 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -14,8 +14,7 @@ jobs: run: dart pub get - name: Run tests - run: | - make test-with-coverage + run: make test - name: Upload coverage to codecov run: | diff --git a/.gitignore b/.gitignore index d15afb9..41b3466 100644 --- a/.gitignore +++ b/.gitignore @@ -74,10 +74,5 @@ example/ios/Flutter/flutter_export_environment.sh !**/ios/**/default.perspectivev3 !/packages/flutter_tools/test/data/dart_dependencies_test/**/.packages -# Coverage report -test/.test_coverage.dart +# Coverage /coverage/ -/coverage-html/ - -# Test output -/test/output/ diff --git a/.pubignore b/.pubignore index ebba33a..8b1c730 100644 --- a/.pubignore +++ b/.pubignore @@ -1,4 +1,3 @@ /test/ Makefile -/coverage/ -/coverage-html/ \ No newline at end of file +/coverage/ \ No newline at end of file diff --git a/Makefile b/Makefile index e6477c3..df13ca5 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,27 @@ SHELL=/bin/bash -test-with-coverage: +# $$$$$$$$$ Testing $$$$$$$$$$ + +test-and-show-coverage: test coverage-html + +run-tests: dart pub global activate coverage dart pub global run coverage:test_with_coverage -coverage: - genhtml coverage/lcov.info -o coverage-html - firefox coverage-html/index.html \ No newline at end of file +coverage-html: + genhtml coverage/lcov.info -o coverage/html + firefox coverage/html/index.html + +# $$$$$$$$$ Releasing $$$$$$$$$$ + +pre-release: + dart format lib + dart analyze + dart pub publish --dry-run + +patch-version: + dart pub global activate pubversion + pubversion patch + +release: + dart pub publish \ No newline at end of file diff --git a/README.md b/README.md index 524c6fb..c44ffac 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ final fromCsv = Dataframe.fromCsv( path: 'path/to/file.csv', eolToken: '\n', maxRows: 40, - skipColumns: ['date'], + skipColumns: ['some-irrelevant-column'], convertDates: true, datePattern: 'dd-MM-yyyy' ); diff --git a/lib/src/column.dart b/lib/src/column.dart index 8f5f38b..ec5b77e 100644 --- a/lib/src/column.dart +++ b/lib/src/column.dart @@ -4,28 +4,26 @@ import 'list_extensions/extended_list_base.dart'; typedef Mask = List; -extension MaskExtensions on Mask{ - Mask operator&(Mask other) => +extension MaskExtensions on Mask { + Mask operator &(Mask other) => IterableZip([this, other]).map((e) => e.first && e.last).toList(); - Mask operator|(Mask other) => + Mask operator |(Mask other) => IterableZip([this, other]).map((e) => e.first | e.last).toList(); - Mask operator^(Mask other) => + Mask operator ^(Mask other) => IterableZip([this, other]).map((e) => e.first ^ e.last).toList(); } -class Column extends ExtendedListBase{ - Column(List records): super(records); +class Column extends ExtendedListBase { + Column(List records) : super(records); - Column cast() => - Column(super.cast()); + Column cast() => Column(super.cast()); // ************* count ***************** /// Count number of occurrences of [element] of the column [colName]. - int count(E object) => - where((element) => element == object).length; + int count(E object) => where((element) => element == object).length; /// Count number of occurrences of values, corresponding to the column [colName], /// equaling any element contained by [pool]. @@ -37,35 +35,31 @@ class Column extends ExtendedListBase{ Column nullFreed({E? replaceWith = null}) => Column(nullFreedIterable(replaceWith: replaceWith).toList()); - Iterable nullFreedIterable({E? replaceWith = null}) => - replaceWith == null - ? where((element) => element != null) - : map((e) => e ?? replaceWith); + Iterable nullFreedIterable({E? replaceWith = null}) => replaceWith == null + ? where((element) => element != null) + : map((e) => e ?? replaceWith); // ****************** transformation ****************** List cumSum() => _nullFreedNums().fold( [], - (sums, element) => - sums..add(sums.isEmpty ? element : sums.last + element)); + (sums, element) => + sums..add(sums.isEmpty ? element : sums.last + element)); // **************** accumulation **************** double mean({bool treatNullsAsZeros = true}) => - _nullFreedNums(treatNullsAsZeros: treatNullsAsZeros).average; + _nullFreedNums(treatNullsAsZeros: treatNullsAsZeros).average; - num max() => - _nullFreedNums().max; + num max() => _nullFreedNums().max; - num min() => - _nullFreedNums().min; + num min() => _nullFreedNums().min; - num sum() => - _nullFreedNums().sum; + num sum() => _nullFreedNums().sum; - Iterable _nullFreedNums({bool treatNullsAsZeros = false}) => - cast().nullFreedIterable(replaceWith: treatNullsAsZeros ? 0.0 : null) - .cast(); + Iterable _nullFreedNums({bool treatNullsAsZeros = false}) => cast() + .nullFreedIterable(replaceWith: treatNullsAsZeros ? 0.0 : null) + .cast(); // ***************** masks ******************* @@ -81,20 +75,19 @@ class Column extends ExtendedListBase{ Mask isNotIn(Set pool) => map((element) => !pool.contains(element)).toList().cast(); - Mask toMask(bool Function(E) test) => - map(test).toList().cast(); + Mask toMask(bool Function(E) test) => map(test).toList().cast(); // ****************** numerical column masks ********************* - Mask operator<(num reference) => + Mask operator <(num reference) => cast().map((element) => element < reference).toList().cast(); - Mask operator>(num reference) => + Mask operator >(num reference) => cast().map((element) => element > reference).toList().cast(); - Mask operator<=(num reference) => + Mask operator <=(num reference) => cast().map((element) => element <= reference).toList().cast(); - Mask operator>=(num reference) => + Mask operator >=(num reference) => cast().map((element) => element >= reference).toList().cast(); -} \ No newline at end of file +} diff --git a/lib/src/dataframe.dart b/lib/src/dataframe.dart index ef0b6c7..d577c54 100644 --- a/lib/src/dataframe.dart +++ b/lib/src/dataframe.dart @@ -245,15 +245,12 @@ class DataFrame extends ExtendedListBase { return Column(column.toList()); } - DataFrame fromColumns(List columnNames) => - DataFrame._copied( - PositionTrackingList(columnNames), - columnNames.map((e) => this(e)).transposed() - ); + DataFrame fromColumns(List columnNames) => DataFrame._copied( + PositionTrackingList(columnNames), + columnNames.map((e) => this(e)).transposed()); /// Returns an iterable over the column data. - Iterable columns() => - _columnNames.map((e) => this(e)); + Iterable columns() => _columnNames.map((e) => this(e)); /// Grab a (typed) record sitting at dataframe[rowIndex][colName]. T record(int rowIndex, String colName) => @@ -263,7 +260,7 @@ class DataFrame extends ExtendedListBase { DataFrame._copied(_columnNames, indices.map((e) => this[e]).toList()); DataFrame rowsWhere(List mask) => - DataFrame._copied(_columnNames, applyMask(mask).toList()); + DataFrame._copied(_columnNames, applyMask(mask).toList()); // **************** manipulation ****************** @@ -355,7 +352,7 @@ class DataFrame extends ExtendedListBase { DataFrame sortedBy(String colName, {bool ascending = true, bool nullsFirst = true, - Comparator? compareRecords}) => + Comparator? compareRecords}) => DataFrame._copied( _columnNames, _sort(colName, @@ -448,4 +445,4 @@ class DataFrame extends ExtendedListBase { .join(consecutiveElementDelimiter)) ]).map((e) => e.join(indexColumnDelimiter)).join('\n'); } -} \ No newline at end of file +} diff --git a/lib/src/utils/iterable.dart b/lib/src/utils/iterable.dart index ac9d038..943bd98 100644 --- a/lib/src/utils/iterable.dart +++ b/lib/src/utils/iterable.dart @@ -7,7 +7,7 @@ extension IterableExtensions on Iterable { whereIndexed((index, _) => mask[index]); } -extension IterableIterableExtensions on Iterable>{ +extension IterableIterableExtensions on Iterable> { List> transposed() => IterableZip(this).map((e) => e.toList()).toList(); } diff --git a/pubspec.yaml b/pubspec.yaml index 44b2827..8d47c1b 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: koala description: A poor man's version of a pandas Dataframe. Collect, access & manipulate related data. -homepage: https://github.com/w2sv/koala +repository: https://github.com/w2sv/koala version: 0.0.3 environment: @@ -13,4 +13,4 @@ dependencies: dev_dependencies: coverage: ^1.6.0 - test: ^1.21.5 + test: ^1.21.5 \ No newline at end of file diff --git a/test/df_test.dart b/test/df_test.dart index c65f2a4..4676a92 100644 --- a/test/df_test.dart +++ b/test/df_test.dart @@ -3,16 +3,13 @@ import 'dart:io'; import 'package:koala/koala.dart'; import 'package:test/test.dart'; -String _csvPath(String name) => - 'test/data/$name'; +String _csvPath(String name) => 'test/data/$name'; -extension RecordsExtensions on Records{ - Set _types() => - map((e) => e.runtimeType).toSet(); +extension RecordsExtensions on Records { + Set _types() => map((e) => e.runtimeType).toSet(); } -DataFrame _getDF() => - DataFrame.fromRowMaps([ +DataFrame _getDF() => DataFrame.fromRowMaps([ {'col1': 1, 'col2': 2}, {'col1': 1, 'col2': 1}, {'col1': null, 'col2': 8}, @@ -20,8 +17,7 @@ DataFrame _getDF() => final _outputDir = Directory('test/output'); -String _outputFilePath(String name) => - '${_outputDir.path}/$name'; +String _outputFilePath(String name) => '${_outputDir.path}/$name'; void main() { test('fromRowMaps', () async { @@ -49,46 +45,64 @@ void main() { group('fromCsv', () { test('basic parsing', () async { - DataFrame df = await DataFrame.fromCsv(path: _csvPath('with_date.csv'), convertDates: false, eolToken: '\n'); - expect(df.columnNames, ['symbol','date','price','n']); + DataFrame df = await DataFrame.fromCsv( + path: _csvPath('with_date.csv'), convertDates: false, eolToken: '\n'); + expect(df.columnNames, ['symbol', 'date', 'price', 'n']); expect(df.length, 2); expect(df('price')._types(), {double}); expect(df('n')._types(), {int}); }); test('automatic date conversion', () async { - DataFrame df = await DataFrame.fromCsv(path: _csvPath('iso_date.csv'), eolToken: '\n'); + DataFrame df = await DataFrame.fromCsv( + path: _csvPath('iso_date.csv'), eolToken: '\n'); expect(df('date')._types(), {DateTime}); - expect(df('date').map((el) => el.toString()).toList(), ['2020-04-12 12:16:54.220', '2020-04-12 12:16:54.220']); + expect(df('date').map((el) => el.toString()).toList(), + ['2020-04-12 12:16:54.220', '2020-04-12 12:16:54.220']); }); test('date conversion with specified format', () async { - DataFrame df = await DataFrame.fromCsv(path: _csvPath('with_date.csv'), eolToken: '\n', datePattern: 'MMM d yyyy'); + DataFrame df = await DataFrame.fromCsv( + path: _csvPath('with_date.csv'), + eolToken: '\n', + datePattern: 'MMM d yyyy'); expect(df('date')._types(), {DateTime}); }); test('newline at the end of file', () async { - DataFrame df = await DataFrame.fromCsv(path: _csvPath('terminating_newline.csv'), eolToken: '\n'); + DataFrame df = await DataFrame.fromCsv( + path: _csvPath('terminating_newline.csv'), eolToken: '\n'); expect(df.length, 1); }); test('max rows', () async { - DataFrame df = await DataFrame.fromCsv(path: _csvPath('stocks.csv'), eolToken: '\n', maxRows: 20); + DataFrame df = await DataFrame.fromCsv( + path: _csvPath('stocks.csv'), eolToken: '\n', maxRows: 20); expect(df.length, 20); }); test('no header', () async { - DataFrame df = await DataFrame.fromCsv(path: _csvPath('no_header.csv'), eolToken: '\n', containsHeader: false, columnNames: ['symbol','date','price','n']); + DataFrame df = await DataFrame.fromCsv( + path: _csvPath('no_header.csv'), + eolToken: '\n', + containsHeader: false, + columnNames: ['symbol', 'date', 'price', 'n']); expect(df.length, 2); - expect(df.columnNames, ['symbol','date','price','n']); + expect(df.columnNames, ['symbol', 'date', 'price', 'n']); }); test('skip columns', () async { - DataFrame df = await DataFrame.fromCsv(path: _csvPath('with_date.csv'), eolToken: '\n', skipColumns: ['price']); + DataFrame df = await DataFrame.fromCsv( + path: _csvPath('with_date.csv'), + eolToken: '\n', + skipColumns: ['price']); expect(df.length, 2); - expect(df.columnNames, ['symbol','date','n']); - expect(df, [['MSFT', 'Jan 1 2000', 1], ['MSFT', 'Feb 1 2000', 2]]); + expect(df.columnNames, ['symbol', 'date', 'n']); + expect(df, [ + ['MSFT', 'Jan 1 2000', 1], + ['MSFT', 'Feb 1 2000', 2] + ]); }); }); @@ -98,35 +112,46 @@ void main() { expect(df.nColumns, 2); }); - test('copying', (){ + test('copying', () { final df = _getDF(); final copy = df.copy()..removeLast(); expect(df == copy, false); expect(df.length == 3 && copy.length == 2, true); }); - test('object overrides', (){ + test('object overrides', () { final df = _getDF(); final df1 = _getDF()..columnNames.add('col3'); expect(df.hashCode == df1.hashCode, false); expect(df == df1, false); - expect(df.toString(), - ' col1 col2\n' - '0 | 1 2 \n' - '1 | 1 1 \n' - '2 | null 8 '); - - final df_with_longer_elements_than_column_names = DataFrame.fromNamesAndData(['a', 'b'], [[888, 1], [null, 8972]]); - expect(df_with_longer_elements_than_column_names.toString(), - ' a b \n' - '0 | 888 1 \n' - '1 | null 8972'); + expect( + df.toString(), + ' col1 col2\n' + '0 | 1 2 \n' + '1 | 1 1 \n' + '2 | null 8 '); + + final df_with_longer_elements_than_column_names = + DataFrame.fromNamesAndData([ + 'a', + 'b' + ], [ + [888, 1], + [null, 8972] + ]); + expect( + df_with_longer_elements_than_column_names.toString(), + ' a b \n' + '0 | 888 1 \n' + '1 | null 8972'); }); test('slicing', () async { - DataFrame df = (await DataFrame.fromCsv(path: _csvPath('stocks.csv'), eolToken: '\n'))..slice(0, 30); + DataFrame df = + (await DataFrame.fromCsv(path: _csvPath('stocks.csv'), eolToken: '\n')) + ..slice(0, 30); expect(df.length, 30); final sliced = df.sliced(5, 25); @@ -135,7 +160,7 @@ void main() { // Ensure disentanglement of copied properties expect(df.length, 30); sliced.removeColumn('symbol'); - expect(df.columnNames, ['symbol','date','price']); + expect(df.columnNames, ['symbol', 'date', 'price']); }); test('mutate', () async { @@ -154,13 +179,10 @@ void main() { // add and remove column df.addColumn('col3', [5, 3]); - expect( - df.rowMaps(), - [ - {'col1': 0, 'col2': 4, 'col3': 5}, - {'col1': 1, 'col2': 2, 'col3': 3} - ] - ); + expect(df.rowMaps(), [ + {'col1': 0, 'col2': 4, 'col3': 5}, + {'col1': 1, 'col2': 2, 'col3': 3} + ]); df.removeColumn('col3'); expect(df.rowMaps(), rows); @@ -187,8 +209,11 @@ void main() { expect(df('col1').runtimeType.toString(), 'List'); // columnIterable - expect(df.columns().toList(), [[1, 1, null], [2, 1, 8]]); - + expect(df.columns().toList(), [ + [1, 1, null], + [2, 1, 8] + ]); + df.rowsWhere((df('col1') > 6) & (df('col2') <= 5)); // record @@ -233,7 +258,8 @@ void main() { {'col1': null, 'col2': null}, {'col1': 3, 'col2': 'b'}, {'col1': 4, 'col2': 'a'}, - ])..sortBy('col2'); + ]) + ..sortBy('col2'); const col1PostSort = [null, 4, 3, 2, 1]; @@ -253,7 +279,7 @@ void main() { final df5 = df1.sortedBy('col2', ascending: false, nullsFirst: false); expect(df5('col2'), [null, 'd', 'c', 'b', 'a']); - + final df6 = df1.sortedBy('col1', compareRecords: (a, b) => 1); expect(df6('col1'), [1, 2, 3, 4, null]); @@ -264,16 +290,31 @@ void main() { group('toCsv', () { test('default', () async { final outputCsvPath = _outputFilePath('out.csv'); - final df = DataFrame.fromNamesAndData(['a', 'b', 'c'], [[12, 'asdf', 33.53], [65, 'dsafa', 89]]); + final df = DataFrame.fromNamesAndData([ + 'a', + 'b', + 'c' + ], [ + [12, 'asdf', 33.53], + [65, 'dsafa', 89] + ]); df.toCsv(outputCsvPath); expect(await DataFrame.fromCsv(path: outputCsvPath), df); }); test('with null', () async { final outputCsvPath = _outputFilePath('out1.csv'); - final df = DataFrame.fromNamesAndData(['a', 'b', 'c'], [[12, 'asdf', null], [null, 'dsafa', 89]]); + final df = DataFrame.fromNamesAndData([ + 'a', + 'b', + 'c' + ], [ + [12, 'asdf', null], + [null, 'dsafa', 89] + ]); df.toCsv(outputCsvPath, nullRepresentation: ''); - expect(await DataFrame.fromCsv(path: outputCsvPath, parseAsNull: {''}), df); + expect( + await DataFrame.fromCsv(path: outputCsvPath, parseAsNull: {''}), df); }); // test('with double quote including strings', () async { @@ -285,14 +326,28 @@ void main() { test('with single quote including strings', () async { final outputCsvPath = _outputFilePath('out3.csv'); - final df = DataFrame.fromNamesAndData(['a', 'b', 'c'], [[12, "as''df", 33.53], [65, "ds'afa", 89]]); + final df = DataFrame.fromNamesAndData([ + 'a', + 'b', + 'c' + ], [ + [12, "as''df", 33.53], + [65, "ds'afa", 89] + ]); df.toCsv(outputCsvPath); expect(await DataFrame.fromCsv(path: outputCsvPath), df); }); test('without header', () async { final outputCsvPath = _outputFilePath('out4.csv'); - final df = DataFrame.fromNamesAndData(['a', 'b', 'c'], [[12, "asdf", 33.53], [65, "dsafa", 89]]); + final df = DataFrame.fromNamesAndData([ + 'a', + 'b', + 'c' + ], [ + [12, "asdf", 33.53], + [65, "dsafa", 89] + ]); df.toCsv(outputCsvPath, includeHeader: false); final lines = await File(outputCsvPath).readAsLines(); @@ -301,16 +356,22 @@ void main() { }); }); - tearDownAll((){ + tearDownAll(() { _outputDir.list().forEach((element) => element.delete()); }); - test('misc', (){ + test('misc', () { final df = DataFrame.empty(); expect(df.nColumns, 0); expect(df.length, 0); expect(() => DataFrame.fromNamesAndData(['b'], []), throwsArgumentError); - expect(() => DataFrame.fromNamesAndData(['b'], [[888, 1]]), throwsArgumentError); + expect( + () => DataFrame.fromNamesAndData([ + 'b' + ], [ + [888, 1] + ]), + throwsArgumentError); }); -} \ No newline at end of file +} diff --git a/test/iterable_test.dart b/test/iterable_test.dart index cb310be..d5582cd 100644 --- a/test/iterable_test.dart +++ b/test/iterable_test.dart @@ -2,8 +2,17 @@ import 'package:koala/src/utils/iterable.dart'; import 'package:test/expect.dart'; import 'package:test/scaffolding.dart'; -void main(){ - test('transposed', (){ - expect([[1, 2, 3], [2, 3, 4]].transposed(), [[1, 2], [2, 3], [3, 4]]); +void main() { + test('transposed', () { + expect( + [ + [1, 2, 3], + [2, 3, 4] + ].transposed(), + [ + [1, 2], + [2, 3], + [3, 4] + ]); }); -} \ No newline at end of file +} diff --git a/test/position_tracking_list_test.dart b/test/position_tracking_list_test.dart index 8352b25..520e1f4 100644 --- a/test/position_tracking_list_test.dart +++ b/test/position_tracking_list_test.dart @@ -2,8 +2,8 @@ import 'package:koala/src/list_extensions/position_tracking_list.dart'; import 'package:koala/src/utils/list.dart'; import 'package:test/test.dart'; -void main(){ - test('PositionTrackingList', (){ +void main() { + test('PositionTrackingList', () { final rawColumns = ['a', 'b', 'c']; final columns = PositionTrackingList(copy1D(rawColumns)); @@ -30,4 +30,4 @@ void main(){ expect(columns.indexOf('e'), 3); expect(columns.indexOf('f'), 4); }); -} \ No newline at end of file +}