@@ -50,7 +50,7 @@ async def download_metadata_schemas(temp_dir: Path) -> list[str]:
5050    }
5151    if  token :
5252        headers ["Authorization" ] =  f"Bearer { token }  
53-      
53+ 
5454    async  with  httpx .AsyncClient (headers = headers , timeout = 30.0 ) as  client :
5555        try :
5656            response  =  await  client .get (METADATA_SCHEMAS_GITHUB_URL )
@@ -64,21 +64,21 @@ async def download_metadata_schemas(temp_dir: Path) -> list[str]:
6464                )
6565                raise 
6666            raise 
67-          
67+ 
6868        yaml_files  =  []
6969        for  file_info  in  files_info :
7070            if  file_info ["name" ].endswith (".yaml" ):
7171                file_name  =  file_info ["name" ]
7272                file_url  =  f"{ METADATA_SCHEMAS_RAW_URL_BASE } { file_name }  
73-                  
73+ 
7474                print (f"Downloading { file_name }  , file = sys .stderr )
7575                file_response  =  await  client .get (file_url )
7676                file_response .raise_for_status ()
77-                  
77+ 
7878                file_path  =  temp_dir  /  file_name 
7979                file_path .write_text (file_response .text )
8080                yaml_files .append (Path (file_name ).stem )
81-          
81+ 
8282        return  yaml_files 
8383
8484
@@ -173,11 +173,9 @@ async def post_process_metadata_models(codegen_container: dagger.Container):
173173            original_content  =  await  codegen_container .file (
174174                f"/generated/{ generated_file }  
175175            ).contents ()
176-             
177-             post_processed_content  =  original_content .replace (
178-                 "from pydantic" , "from pydantic.v1" 
179-             )
180-             
176+ 
177+             post_processed_content  =  original_content .replace ("from pydantic" , "from pydantic.v1" )
178+ 
181179            codegen_container  =  codegen_container .with_new_file (
182180                f"/generated_post_processed/{ generated_file }  , contents = post_processed_content 
183181            )
@@ -194,7 +192,7 @@ async def generate_models_from_schemas(
194192) ->  None :
195193    """Generate Pydantic models from YAML schemas using datamodel-codegen.""" 
196194    init_module_content  =  generate_init_module_content (yaml_files )
197-      
195+ 
198196    codegen_container  =  (
199197        dagger_client .container ()
200198        .from_ (PYTHON_IMAGE )
@@ -205,7 +203,7 @@ async def generate_models_from_schemas(
205203        )
206204        .with_new_file ("/generated/__init__.py" , contents = init_module_content )
207205    )
208-      
206+ 
209207    for  yaml_file  in  yaml_files :
210208        codegen_container  =  codegen_container .with_exec (
211209            [
@@ -226,7 +224,7 @@ async def generate_models_from_schemas(
226224            ],
227225            use_entrypoint = True ,
228226        )
229-      
227+ 
230228    if  post_process :
231229        codegen_container  =  await  post_process_codegen (codegen_container )
232230        await  codegen_container .directory ("/generated_post_processed" ).export (output_dir_path )
@@ -240,35 +238,38 @@ async def generate_models_from_schemas(
240238def  consolidate_yaml_schemas_to_json (yaml_dir_path : Path , output_json_path : str ) ->  None :
241239    """Consolidate all YAML schemas into a single JSON schema file.""" 
242240    schemas  =  {}
243-      
241+ 
244242    for  yaml_file  in  yaml_dir_path .glob ("*.yaml" ):
245243        schema_name  =  yaml_file .stem 
246-         with  yaml_file .open ('r' ) as  f :
244+         with  yaml_file .open ("r" ) as  f :
247245            schema_content  =  yaml .safe_load (f )
248246            schemas [schema_name ] =  schema_content 
249-      
247+ 
250248    # Find the main schema (ConnectorMetadataDefinitionV0) 
251249    main_schema  =  schemas .get ("ConnectorMetadataDefinitionV0" )
252-      
250+ 
253251    if  main_schema :
254252        # Create a consolidated schema with definitions 
255253        consolidated  =  {
256254            "$schema" : main_schema .get ("$schema" , "http://json-schema.org/draft-07/schema#" ),
257255            "title" : "Connector Metadata Schema" ,
258256            "description" : "Consolidated JSON schema for Airbyte connector metadata validation" ,
259257            ** main_schema ,
260-             "definitions" : {}
258+             "definitions" : {}, 
261259        }
262-          
260+ 
263261        # Add all other schemas as definitions 
264262        for  schema_name , schema_content  in  schemas .items ():
265263            if  schema_name  !=  "ConnectorMetadataDefinitionV0" :
266264                consolidated ["definitions" ][schema_name ] =  schema_content 
267-          
265+ 
268266        Path (output_json_path ).write_text (json .dumps (consolidated , indent = 2 ))
269267        print (f"Generated consolidated JSON schema: { output_json_path }  , file = sys .stderr )
270268    else :
271-         print ("Warning: ConnectorMetadataDefinitionV0 not found, generating simple consolidation" , file = sys .stderr )
269+         print (
270+             "Warning: ConnectorMetadataDefinitionV0 not found, generating simple consolidation" ,
271+             file = sys .stderr ,
272+         )
272273        Path (output_json_path ).write_text (json .dumps (schemas , indent = 2 ))
273274
274275
@@ -287,7 +288,7 @@ async def generate_metadata_models_single_file(
287288            "/yaml" , dagger_client .host ().directory (yaml_dir_path , include = ["*.yaml" ])
288289        )
289290    )
290-      
291+ 
291292    codegen_container  =  codegen_container .with_exec (
292293        [
293294            "datamodel-codegen" ,
@@ -307,16 +308,14 @@ async def generate_metadata_models_single_file(
307308        ],
308309        use_entrypoint = True ,
309310    )
310-      
311+ 
311312    original_content  =  await  codegen_container .file ("/generated/models.py" ).contents ()
312-     post_processed_content  =  original_content .replace (
313-         "from pydantic" , "from pydantic.v1" 
314-     )
315-     
313+     post_processed_content  =  original_content .replace ("from pydantic" , "from pydantic.v1" )
314+ 
316315    codegen_container  =  codegen_container .with_new_file (
317316        "/generated/models_processed.py" , contents = post_processed_content 
318317    )
319-      
318+ 
320319    await  codegen_container .file ("/generated/models_processed.py" ).export (output_file_path )
321320
322321
@@ -331,27 +330,27 @@ async def main():
331330            yaml_files = declarative_yaml_files ,
332331            post_process = True ,
333332        )
334-          
333+ 
335334        print ("\n Generating metadata models..." , file = sys .stderr )
336335        with  tempfile .TemporaryDirectory () as  temp_dir :
337336            temp_path  =  Path (temp_dir )
338337            await  download_metadata_schemas (temp_path )
339-              
338+ 
340339            output_dir  =  Path (LOCAL_METADATA_OUTPUT_DIR_PATH )
341340            output_dir .mkdir (parents = True , exist_ok = True )
342-              
341+ 
343342            print ("Generating single Python file with all models..." , file = sys .stderr )
344343            output_file  =  str (output_dir  /  "models.py" )
345344            await  generate_metadata_models_single_file (
346345                dagger_client = dagger_client ,
347346                yaml_dir_path = str (temp_path ),
348347                output_file_path = output_file ,
349348            )
350-              
349+ 
351350            print ("Generating consolidated JSON schema..." , file = sys .stderr )
352351            json_schema_file  =  str (output_dir  /  "metadata_schema.json" )
353352            consolidate_yaml_schemas_to_json (temp_path , json_schema_file )
354-          
353+ 
355354        print ("\n Model generation complete!" , file = sys .stderr )
356355
357356
0 commit comments