Skip to content

Commit

Permalink
Don't track write count in the JSONLinesLoader (#1469)
Browse files Browse the repository at this point in the history
  • Loading branch information
jmortlock authored Feb 12, 2025
1 parent ad2dff4 commit e720fc5
Showing 1 changed file with 2 additions and 20 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@ final class JsonLinesLoader implements Closure, Loader, Loader\FileLoader

private int $flags = JSON_THROW_ON_ERROR;

/**
* @var array<string, int>
*/
private array $writes = [];

public function __construct(private readonly Path $path)
{
}
Expand Down Expand Up @@ -65,16 +60,7 @@ public function write(Rows $nextRows, array $partitions, FlowContext $context) :
$streams = $context->streams();
$normalizer = new RowsNormalizer(new EntryNormalizer($this->dateTimeFormat));

if (!$streams->isOpen($this->path, $partitions)) {
$stream = $streams->writeTo($this->path, $partitions);

if (!\array_key_exists($stream->path()->path(), $this->writes)) {
$this->writes[$stream->path()->path()] = 0;
}

} else {
$stream = $streams->writeTo($this->path, $partitions);
}
$stream = $streams->writeTo($this->path, $partitions);

$this->writeJSON($nextRows, $stream, $normalizer);
}
Expand Down Expand Up @@ -103,11 +89,7 @@ private function writeJSON(Rows $rows, DestinationStream $stream, RowsNormalizer
throw new RuntimeException('Failed to encode JSON: ' . $e->getMessage(), 0, $e);
}

$json = ($this->writes[$stream->path()->path()] > 0) ? ("\n" . $json) : $json;

$stream->append($json);

$this->writes[$stream->path()->path()]++;
$stream->append($json . "\n");
}
}
}

0 comments on commit e720fc5

Please sign in to comment.