merge with master and re-add ordering to python tests

.github/workflows/CodeQuality.yml

@@ -49,13 +49,14 @@ jobs:
 
       - name: Install
         shell: bash
-        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang-format && sudo pip3 install cmake-format
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang-format && sudo pip3 install cmake-format black
 
       - name: Format Check
         shell: bash
         run: |
           clang-format --version
           clang-format --dump-config
+          black --version
           make format-check-silent
 
       - name: Generated Check

examples/python/duckdb-python.py

@@ -30,7 +30,8 @@
 # we can query pandas data frames as if they were SQL views
 # create a sample pandas data frame
 import pandas as pd
-test_df = pd.DataFrame.from_dict({"i":[1, 2, 3, 4], "j":["one", "two", "three", "four"]})
+
+test_df = pd.DataFrame.from_dict({"i": [1, 2, 3, 4], "j": ["one", "two", "three", "four"]})
 
 # make this data frame available as a view in duckdb
 conn.register("test_df", test_df)
@@ -49,8 +50,9 @@
 
 # create a relation from a CSV file
 
-# first create a CSV file from our pandas example 
+# first create a CSV file from our pandas example
 import tempfile, os
+
 temp_file_name = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names()))
 test_df.to_csv(temp_file_name, index=False)
 
@@ -139,9 +141,7 @@
 # turn the relation into something else again
 
 
-
-
-# compute the query result from the relation 
+# compute the query result from the relation
 res = rel.execute()
 print(res)
 # res is a query result, you can call fetchdf() or fetchnumpy() or fetchone() on it
@@ -164,7 +164,7 @@
 # Inserting elements into table_3
 print(conn.values([5, 'five']).insert_into("test_table3"))
 rel_3 = conn.table("test_table3")
-rel_3.insert([6,'six'])
+rel_3.insert([6, 'six'])
 
 # create a SQL-accessible view of the relation
 print(rel.create_view('test_view'))
@@ -183,6 +183,3 @@
 # this also works directly on data frames
 res = duckdb.query(test_df, 'my_name_for_test_df', 'SELECT * FROM my_name_for_test_df')
 print(res.df())
-
-
-

extension/excel/excel_config.py

@@ -1,6 +1,16 @@
 import os
+
 # list all include directories
-include_directories = [os.path.sep.join(x.split('/')) for x in ['extension/excel/include', 'extension/excel/numformat/include']]
+include_directories = [
+    os.path.sep.join(x.split('/')) for x in ['extension/excel/include', 'extension/excel/numformat/include']
+]
 # source files
 source_files = [os.path.sep.join(x.split('/')) for x in ['extension/excel/excel_extension.cpp']]
-source_files += [os.path.sep.join(x.split('/')) for x in ['extension/excel/numformat/nf_calendar.cpp', 'extension/excel/numformat/nf_localedata.cpp', 'extension/excel/numformat/nf_zformat.cpp']]
+source_files += [
+    os.path.sep.join(x.split('/'))
+    for x in [
+        'extension/excel/numformat/nf_calendar.cpp',
+        'extension/excel/numformat/nf_localedata.cpp',
+        'extension/excel/numformat/nf_zformat.cpp',
+    ]
+]

extension/fts/fts_config.py

@@ -1,8 +1,55 @@
 import os
+
 # list all include directories
-include_directories = [os.path.sep.join(x.split('/')) for x in ['extension/fts/include', 'third_party/snowball/libstemmer', 'third_party/snowball/runtime', 'third_party/snowball/src_c']]
+include_directories = [
+    os.path.sep.join(x.split('/'))
+    for x in [
+        'extension/fts/include',
+        'third_party/snowball/libstemmer',
+        'third_party/snowball/runtime',
+        'third_party/snowball/src_c',
+    ]
+]
 # source files
-source_files = [os.path.sep.join(x.split('/')) for x in ['extension/fts/fts_extension.cpp', 'extension/fts/fts_indexing.cpp']]
+source_files = [
+    os.path.sep.join(x.split('/')) for x in ['extension/fts/fts_extension.cpp', 'extension/fts/fts_indexing.cpp']
+]
 # snowball
-source_files += [os.path.sep.join(x.split('/')) for x in ['third_party/snowball/libstemmer/libstemmer.cpp', 'third_party/snowball/runtime/utilities.cpp', 'third_party/snowball/runtime/api.cpp', 'third_party/snowball/src_c/stem_UTF_8_arabic.cpp', 'third_party/snowball/src_c/stem_UTF_8_basque.cpp', 'third_party/snowball/src_c/stem_UTF_8_catalan.cpp', 'third_party/snowball/src_c/stem_UTF_8_danish.cpp', 'third_party/snowball/src_c/stem_UTF_8_dutch.cpp', 'third_party/snowball/src_c/stem_UTF_8_english.cpp', 'third_party/snowball/src_c/stem_UTF_8_finnish.cpp', 'third_party/snowball/src_c/stem_UTF_8_french.cpp', 'third_party/snowball/src_c/stem_UTF_8_german.cpp', 'third_party/snowball/src_c/stem_UTF_8_german2.cpp', 'third_party/snowball/src_c/stem_UTF_8_greek.cpp', 'third_party/snowball/src_c/stem_UTF_8_hindi.cpp', 'third_party/snowball/src_c/stem_UTF_8_hungarian.cpp', 'third_party/snowball/src_c/stem_UTF_8_indonesian.cpp', 'third_party/snowball/src_c/stem_UTF_8_irish.cpp', 'third_party/snowball/src_c/stem_UTF_8_italian.cpp', 'third_party/snowball/src_c/stem_UTF_8_kraaij_pohlmann.cpp', 'third_party/snowball/src_c/stem_UTF_8_lithuanian.cpp', 'third_party/snowball/src_c/stem_UTF_8_lovins.cpp', 'third_party/snowball/src_c/stem_UTF_8_nepali.cpp', 'third_party/snowball/src_c/stem_UTF_8_norwegian.cpp', 'third_party/snowball/src_c/stem_UTF_8_porter.cpp', 'third_party/snowball/src_c/stem_UTF_8_portuguese.cpp', 'third_party/snowball/src_c/stem_UTF_8_romanian.cpp', 'third_party/snowball/src_c/stem_UTF_8_russian.cpp', 'third_party/snowball/src_c/stem_UTF_8_serbian.cpp', 'third_party/snowball/src_c/stem_UTF_8_spanish.cpp', 'third_party/snowball/src_c/stem_UTF_8_swedish.cpp', 'third_party/snowball/src_c/stem_UTF_8_tamil.cpp', 'third_party/snowball/src_c/stem_UTF_8_turkish.cpp']]
-
+source_files += [
+    os.path.sep.join(x.split('/'))
+    for x in [
+        'third_party/snowball/libstemmer/libstemmer.cpp',
+        'third_party/snowball/runtime/utilities.cpp',
+        'third_party/snowball/runtime/api.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_arabic.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_basque.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_catalan.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_danish.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_dutch.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_english.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_finnish.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_french.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_german.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_german2.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_greek.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_hindi.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_hungarian.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_indonesian.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_irish.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_italian.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_kraaij_pohlmann.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_lithuanian.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_lovins.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_nepali.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_norwegian.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_porter.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_portuguese.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_romanian.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_russian.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_serbian.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_spanish.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_swedish.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_tamil.cpp',
+        'third_party/snowball/src_c/stem_UTF_8_turkish.cpp',
+    ]
+]

extension/httpfs/httpfs_config.py

@@ -1,5 +1,12 @@
 import os
+
 # list all include directories
-include_directories = [os.path.sep.join(x.split('/')) for x in ['extension/httpfs/include', 'third_party/httplib', 'extension/parquet/include']]
+include_directories = [
+    os.path.sep.join(x.split('/'))
+    for x in ['extension/httpfs/include', 'third_party/httplib', 'extension/parquet/include']
+]
 # source files
-source_files = [os.path.sep.join(x.split('/')) for x in ['extension/httpfs/' + s for s in ['httpfs_extension.cpp', 'httpfs.cpp', 's3fs.cpp', 'crypto.cpp']]]
+source_files = [
+    os.path.sep.join(x.split('/'))
+    for x in ['extension/httpfs/' + s for s in ['httpfs_extension.cpp', 'httpfs.cpp', 's3fs.cpp', 'crypto.cpp']]
+]

extension/icu/icu_config.py

@@ -1,10 +1,18 @@
 import os
 
 # list all include directories
-include_directories = [os.path.sep.join(x.split('/')) for x in ['extension/icu/include', 'extension/icu/third_party/icu/common', 'extension/icu/third_party/icu/i18n']]
+include_directories = [
+    os.path.sep.join(x.split('/'))
+    for x in ['extension/icu/include', 'extension/icu/third_party/icu/common', 'extension/icu/third_party/icu/i18n']
+]
 # source files
-source_directories = [os.path.sep.join(x.split('/')) for x in ['.', 'third_party/icu/common', 'third_party/icu/i18n', 'third_party/icu/stubdata']]
+source_directories = [
+    os.path.sep.join(x.split('/'))
+    for x in ['.', 'third_party/icu/common', 'third_party/icu/i18n', 'third_party/icu/stubdata']
+]
 source_files = []
 base_path = os.path.dirname(os.path.abspath(__file__))
 for dir in source_directories:
-    source_files += [os.path.join('extension', 'icu', dir, x) for x in os.listdir(os.path.join(base_path, dir)) if x.endswith('.cpp')]
+    source_files += [
+        os.path.join('extension', 'icu', dir, x) for x in os.listdir(os.path.join(base_path, dir)) if x.endswith('.cpp')
+    ]

extension/icu/scripts/inline-data.py

@@ -15,6 +15,8 @@
 extern "C" U_EXPORT const unsigned char U_ICUDATA_ENTRY_POINT [] = {
 	%s
 };
-""" % (result_text,)
+""" % (
+    result_text,
+)
 
 sys.stdout.write(new_contents)

extension/jemalloc/jemalloc_config.py

@@ -1,5 +1,75 @@
 import os
+
 # list all include directories
-include_directories = [os.path.sep.join(x.split('/')) for x in ['extension/jemalloc/include', 'extension/jemalloc/jemalloc/include']]
+include_directories = [
+    os.path.sep.join(x.split('/')) for x in ['extension/jemalloc/include', 'extension/jemalloc/jemalloc/include']
+]
 # source files
-source_files = [os.path.sep.join(x.split('/')) for x in ['extension/jemalloc/jemalloc_extension.cpp', 'extension/jemalloc/jemalloc/src/arena.cpp', 'extension/jemalloc/jemalloc/src/background_thread.cpp', 'extension/jemalloc/jemalloc/src/base.cpp', 'extension/jemalloc/jemalloc/src/bin.cpp', 'extension/jemalloc/jemalloc/src/bin_info.cpp', 'extension/jemalloc/jemalloc/src/bitmap.cpp', 'extension/jemalloc/jemalloc/src/buf_writer.cpp', 'extension/jemalloc/jemalloc/src/cache_bin.cpp', 'extension/jemalloc/jemalloc/src/ckh.cpp', 'extension/jemalloc/jemalloc/src/counter.cpp', 'extension/jemalloc/jemalloc/src/ctl.cpp', 'extension/jemalloc/jemalloc/src/decay.cpp', 'extension/jemalloc/jemalloc/src/div.cpp', 'extension/jemalloc/jemalloc/src/ecache.cpp', 'extension/jemalloc/jemalloc/src/edata.cpp', 'extension/jemalloc/jemalloc/src/edata_cache.cpp', 'extension/jemalloc/jemalloc/src/ehooks.cpp', 'extension/jemalloc/jemalloc/src/emap.cpp', 'extension/jemalloc/jemalloc/src/eset.cpp', 'extension/jemalloc/jemalloc/src/exp_grow.cpp', 'extension/jemalloc/jemalloc/src/extent.cpp', 'extension/jemalloc/jemalloc/src/extent_dss.cpp', 'extension/jemalloc/jemalloc/src/extent_mmap.cpp', 'extension/jemalloc/jemalloc/src/fxp.cpp', 'extension/jemalloc/jemalloc/src/hook.cpp', 'extension/jemalloc/jemalloc/src/hpa.cpp', 'extension/jemalloc/jemalloc/src/hpa_hooks.cpp', 'extension/jemalloc/jemalloc/src/hpdata.cpp', 'extension/jemalloc/jemalloc/src/inspect.cpp', 'extension/jemalloc/jemalloc/src/jemalloc.cpp', 'extension/jemalloc/jemalloc/src/large.cpp', 'extension/jemalloc/jemalloc/src/log.cpp', 'extension/jemalloc/jemalloc/src/malloc_io.cpp', 'extension/jemalloc/jemalloc/src/mutex.cpp', 'extension/jemalloc/jemalloc/src/nstime.cpp', 'extension/jemalloc/jemalloc/src/pa.cpp', 'extension/jemalloc/jemalloc/src/pa_extra.cpp', 'extension/jemalloc/jemalloc/src/pac.cpp', 'extension/jemalloc/jemalloc/src/pages.cpp', 'extension/jemalloc/jemalloc/src/pai.cpp', 'extension/jemalloc/jemalloc/src/peak_event.cpp', 'extension/jemalloc/jemalloc/src/prof.cpp', 'extension/jemalloc/jemalloc/src/prof_data.cpp', 'extension/jemalloc/jemalloc/src/prof_log.cpp', 'extension/jemalloc/jemalloc/src/prof_recent.cpp', 'extension/jemalloc/jemalloc/src/prof_stats.cpp', 'extension/jemalloc/jemalloc/src/prof_sys.cpp', 'extension/jemalloc/jemalloc/src/psset.cpp', 'extension/jemalloc/jemalloc/src/rtree.cpp', 'extension/jemalloc/jemalloc/src/safety_check.cpp', 'extension/jemalloc/jemalloc/src/san.cpp', 'extension/jemalloc/jemalloc/src/san_bump.cpp', 'extension/jemalloc/jemalloc/src/sc.cpp', 'extension/jemalloc/jemalloc/src/sec.cpp', 'extension/jemalloc/jemalloc/src/stats.cpp', 'extension/jemalloc/jemalloc/src/sz.cpp', 'extension/jemalloc/jemalloc/src/tcache.cpp', 'extension/jemalloc/jemalloc/src/test_hooks.cpp', 'extension/jemalloc/jemalloc/src/thread_event.cpp', 'extension/jemalloc/jemalloc/src/ticker.cpp', 'extension/jemalloc/jemalloc/src/tsd.cpp', 'extension/jemalloc/jemalloc/src/witness.cpp']]
+source_files = [
+    os.path.sep.join(x.split('/'))
+    for x in [
+        'extension/jemalloc/jemalloc_extension.cpp',
+        'extension/jemalloc/jemalloc/src/arena.cpp',
+        'extension/jemalloc/jemalloc/src/background_thread.cpp',
+        'extension/jemalloc/jemalloc/src/base.cpp',
+        'extension/jemalloc/jemalloc/src/bin.cpp',
+        'extension/jemalloc/jemalloc/src/bin_info.cpp',
+        'extension/jemalloc/jemalloc/src/bitmap.cpp',
+        'extension/jemalloc/jemalloc/src/buf_writer.cpp',
+        'extension/jemalloc/jemalloc/src/cache_bin.cpp',
+        'extension/jemalloc/jemalloc/src/ckh.cpp',
+        'extension/jemalloc/jemalloc/src/counter.cpp',
+        'extension/jemalloc/jemalloc/src/ctl.cpp',
+        'extension/jemalloc/jemalloc/src/decay.cpp',
+        'extension/jemalloc/jemalloc/src/div.cpp',
+        'extension/jemalloc/jemalloc/src/ecache.cpp',
+        'extension/jemalloc/jemalloc/src/edata.cpp',
+        'extension/jemalloc/jemalloc/src/edata_cache.cpp',
+        'extension/jemalloc/jemalloc/src/ehooks.cpp',
+        'extension/jemalloc/jemalloc/src/emap.cpp',
+        'extension/jemalloc/jemalloc/src/eset.cpp',
+        'extension/jemalloc/jemalloc/src/exp_grow.cpp',
+        'extension/jemalloc/jemalloc/src/extent.cpp',
+        'extension/jemalloc/jemalloc/src/extent_dss.cpp',
+        'extension/jemalloc/jemalloc/src/extent_mmap.cpp',
+        'extension/jemalloc/jemalloc/src/fxp.cpp',
+        'extension/jemalloc/jemalloc/src/hook.cpp',
+        'extension/jemalloc/jemalloc/src/hpa.cpp',
+        'extension/jemalloc/jemalloc/src/hpa_hooks.cpp',
+        'extension/jemalloc/jemalloc/src/hpdata.cpp',
+        'extension/jemalloc/jemalloc/src/inspect.cpp',
+        'extension/jemalloc/jemalloc/src/jemalloc.cpp',
+        'extension/jemalloc/jemalloc/src/large.cpp',
+        'extension/jemalloc/jemalloc/src/log.cpp',
+        'extension/jemalloc/jemalloc/src/malloc_io.cpp',
+        'extension/jemalloc/jemalloc/src/mutex.cpp',
+        'extension/jemalloc/jemalloc/src/nstime.cpp',
+        'extension/jemalloc/jemalloc/src/pa.cpp',
+        'extension/jemalloc/jemalloc/src/pa_extra.cpp',
+        'extension/jemalloc/jemalloc/src/pac.cpp',
+        'extension/jemalloc/jemalloc/src/pages.cpp',
+        'extension/jemalloc/jemalloc/src/pai.cpp',
+        'extension/jemalloc/jemalloc/src/peak_event.cpp',
+        'extension/jemalloc/jemalloc/src/prof.cpp',
+        'extension/jemalloc/jemalloc/src/prof_data.cpp',
+        'extension/jemalloc/jemalloc/src/prof_log.cpp',
+        'extension/jemalloc/jemalloc/src/prof_recent.cpp',
+        'extension/jemalloc/jemalloc/src/prof_stats.cpp',
+        'extension/jemalloc/jemalloc/src/prof_sys.cpp',
+        'extension/jemalloc/jemalloc/src/psset.cpp',
+        'extension/jemalloc/jemalloc/src/rtree.cpp',
+        'extension/jemalloc/jemalloc/src/safety_check.cpp',
+        'extension/jemalloc/jemalloc/src/san.cpp',
+        'extension/jemalloc/jemalloc/src/san_bump.cpp',
+        'extension/jemalloc/jemalloc/src/sc.cpp',
+        'extension/jemalloc/jemalloc/src/sec.cpp',
+        'extension/jemalloc/jemalloc/src/stats.cpp',
+        'extension/jemalloc/jemalloc/src/sz.cpp',
+        'extension/jemalloc/jemalloc/src/tcache.cpp',
+        'extension/jemalloc/jemalloc/src/test_hooks.cpp',
+        'extension/jemalloc/jemalloc/src/thread_event.cpp',
+        'extension/jemalloc/jemalloc/src/ticker.cpp',
+        'extension/jemalloc/jemalloc/src/tsd.cpp',
+        'extension/jemalloc/jemalloc/src/witness.cpp',
+    ]
+]

extension/json/json_config.py

@@ -1,5 +1,34 @@
 import os
+
 # list all include directories
-include_directories = [os.path.sep.join(x.split('/')) for x in ['extension/json/include', 'extension/json/yyjson/include']]
+include_directories = [
+    os.path.sep.join(x.split('/')) for x in ['extension/json/include', 'extension/json/yyjson/include']
+]
 # source files
-source_files = [os.path.sep.join(x.split('/')) for x in ['extension/json/buffered_json_reader.cpp', 'extension/json/json_extension.cpp', 'extension/json/json_common.cpp', 'extension/json/json_functions.cpp', 'extension/json/json_scan.cpp', 'extension/json/json_functions/copy_json.cpp', 'extension/json/json_functions/json_array_length.cpp', 'extension/json/json_functions/json_contains.cpp', 'extension/json/json_functions/json_extract.cpp', 'extension/json/json_functions/json_keys.cpp', 'extension/json/json_functions/json_merge_patch.cpp', 'extension/json/json_functions/json_structure.cpp', 'extension/json/json_functions/json_transform.cpp', 'extension/json/json_functions/json_create.cpp', 'extension/json/json_functions/json_type.cpp', 'extension/json/json_functions/json_valid.cpp', 'extension/json/json_functions/read_json_objects.cpp', 'extension/json/json_functions/read_json.cpp', 'extension/json/yyjson/yyjson.cpp', 'extension/json/json_functions/json_serialize_sql.cpp', 'extension/json/json_serializer.cpp', 'extension/json/json_deserializer.cpp']]
+source_files = [
+    os.path.sep.join(x.split('/'))
+    for x in [
+        'extension/json/buffered_json_reader.cpp',
+        'extension/json/json_extension.cpp',
+        'extension/json/json_common.cpp',
+        'extension/json/json_functions.cpp',
+        'extension/json/json_scan.cpp',
+        'extension/json/json_functions/copy_json.cpp',
+        'extension/json/json_functions/json_array_length.cpp',
+        'extension/json/json_functions/json_contains.cpp',
+        'extension/json/json_functions/json_extract.cpp',
+        'extension/json/json_functions/json_keys.cpp',
+        'extension/json/json_functions/json_merge_patch.cpp',
+        'extension/json/json_functions/json_structure.cpp',
+        'extension/json/json_functions/json_transform.cpp',
+        'extension/json/json_functions/json_create.cpp',
+        'extension/json/json_functions/json_type.cpp',
+        'extension/json/json_functions/json_valid.cpp',
+        'extension/json/json_functions/read_json_objects.cpp',
+        'extension/json/json_functions/read_json.cpp',
+        'extension/json/yyjson/yyjson.cpp',
+        'extension/json/json_functions/json_serialize_sql.cpp',
+        'extension/json/json_serializer.cpp',
+        'extension/json/json_deserializer.cpp',
+    ]
+]