Merge pull request #689 from spotify/update-scio-0.13.6

Update scio to 0.13.6

build.sbt

@@ -20,9 +20,9 @@ import Keys._
 
 val algebirdVersion = "0.13.10"
 val avroVersion = "1.8.2" // keep in sync with scio
-val beamVersion = "2.50.0" // keep in sync with scio
-val bigqueryVersion = "v2-rev20230520-2.0.0" // keep in sync with scio
-val floggerVersion = "0.7.4"
+val beamVersion = "2.52.0" // keep in sync with scio
+val bigqueryVersion = "v2-rev20230812-2.0.0" // keep in sync with scio
+val floggerVersion = "0.7.4" // keep in sync with scio + beam
 val guavaVersion = "32.1.2-jre" // keep in sync with scio + beam
 val hadoopVersion = "2.10.2" // keep in sync with scio
 val jodaTimeVersion = "2.10.10" // keep in sync with scio
@@ -31,7 +31,7 @@ val protoBufVersion = "3.23.2" // keep in sync with scio
 val scalaTestVersion = "3.2.17"
 val scalaCheckVersion = "1.17.0"
 val scalaCollectionCompatVersion = "2.11.0"
-val scioVersion = "0.13.5"
+val scioVersion = "0.13.6"
 val scoptVersion = "4.1.0"
 val shapelessVersion = "2.3.10" // keep in sync with scio
 val sourcecodeVersion = "0.3.1"

ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/AvroDiffyTest.scala

@@ -24,7 +24,7 @@ import com.spotify.ratatool.Schemas
 import com.spotify.ratatool.avro.specific._
 import com.spotify.ratatool.scalacheck._
 import org.apache.avro.generic.{GenericRecord, GenericRecordBuilder}
-import org.apache.beam.sdk.coders.AvroCoder
+import org.apache.beam.sdk.extensions.avro.coders.AvroCoder
 import org.apache.beam.sdk.util.CoderUtils
 import org.scalacheck.Arbitrary
 
@@ -48,7 +48,7 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers {
   }
 
   it should "support nested fields" in {
-    val coder = AvroCoder.of(classOf[TestRecord])
+    val coder = AvroCoder.reflect(classOf[TestRecord])
 
     val nnr = specificRecordOf[NullableNestedRecord].sample.get
     nnr.setIntField(10)
@@ -83,7 +83,7 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers {
   }
 
   it should "support repeated fields" in {
-    val coder = AvroCoder.of(classOf[TestRecord])
+    val coder = AvroCoder.reflect(classOf[TestRecord])
 
     val x = specificRecordOf[TestRecord].sample.get
     x.getRepeatedFields.setIntField(jl(10, 11))
@@ -125,7 +125,7 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers {
   }
 
   it should "support unordered" in {
-    val coder = AvroCoder.of(classOf[TestRecord])
+    val coder = AvroCoder.reflect(classOf[TestRecord])
 
     val a = NullableNestedRecord.newBuilder().setIntField(10).setLongField(100L).build()
     val b = NullableNestedRecord.newBuilder().setIntField(20).setLongField(200L).build()
@@ -146,8 +146,8 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers {
   }
 
   it should "support unordered nested" in {
-    val drnrCoder = AvroCoder.of(classOf[RepeatedRecord])
-    val drrCoder = AvroCoder.of(classOf[DeeplyRepeatedRecord])
+    val drnrCoder = AvroCoder.reflect(classOf[RepeatedRecord])
+    val drrCoder = AvroCoder.reflect(classOf[DeeplyRepeatedRecord])
 
     val a = avroOf[RepeatedRecord].sample.get
     a.setNestedRepeatedField(jl(10, 20, 30))
@@ -176,8 +176,8 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers {
   }
 
   it should "support unordered nested of different lengths" in {
-    val drnrCoder = AvroCoder.of(classOf[RepeatedRecord])
-    val drrCoder = AvroCoder.of(classOf[DeeplyRepeatedRecord])
+    val drnrCoder = AvroCoder.reflect(classOf[RepeatedRecord])
+    val drrCoder = AvroCoder.reflect(classOf[DeeplyRepeatedRecord])
 
     val a = avroOf[RepeatedRecord].sample.get
     a.setNestedRepeatedField(jl(30, 20, 10))

ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/BigDiffyTest.scala

@@ -30,7 +30,7 @@ import com.spotify.ratatool.io.{ParquetIO, ParquetTestData}
 import com.spotify.scio.ScioContext
 import org.apache.avro.Schema
 import org.apache.avro.generic.{GenericData, GenericRecord}
-import org.apache.beam.sdk.coders.AvroCoder
+import org.apache.beam.sdk.extensions.avro.coders.AvroCoder
 import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder
 
 import scala.jdk.CollectionConverters.seqAsJavaListConverter
@@ -39,7 +39,7 @@ import scala.language.higherKinds
 class BigDiffyTest extends PipelineSpec {
 
   val keys = (1 to 1000).map(k => MultiKey("key" + k))
-  val coder = AvroCoder.of(classOf[TestRecord])
+  val coder = AvroCoder.reflect(classOf[TestRecord])
 
   /** Fixed to a small range so that Std. Dev. & Variance calculations are easier to predict */
   val rnr = specificRecordOf[RequiredNestedRecord]

ratatool-examples/src/main/scala/com/spotify/ratatool/examples/diffy/PreProcessBigDiffy.scala

@@ -22,7 +22,7 @@ import com.spotify.ratatool.diffy.{AvroDiffy, BigDiffy, MultiKey}
 import com.spotify.scio.ContextAndArgs
 import org.apache.beam.sdk.util.CoderUtils
 import com.spotify.scio.avro._
-import org.apache.beam.sdk.coders.AvroCoder
+import org.apache.beam.sdk.extensions.avro.coders.AvroCoder
 
 object PreProcessBigDiffy {
   def recordKeyFn(r: ExampleRecord): MultiKey =
@@ -37,7 +37,7 @@ object PreProcessBigDiffy {
   }
 
   def main(cmdlineArgs: Array[String]): Unit = {
-    @transient lazy val coder = AvroCoder.of(classOf[ExampleRecord])
+    @transient lazy val coder = AvroCoder.reflect(classOf[ExampleRecord])
     val (sc, args) = ContextAndArgs(cmdlineArgs)
 
     val (lhsPath, rhsPath, output, header, ignore, unordered) =

ratatool-examples/src/test/scala/com/spotify/ratatool/examples/PreProcessBigDiffyJobTest.scala

@@ -25,15 +25,15 @@ import org.apache.beam.sdk.util.CoderUtils
 import org.scalacheck.Gen
 import com.spotify.scio.avro._
 import com.spotify.scio.io.TextIO
-import org.apache.beam.sdk.coders.AvroCoder
+import org.apache.beam.sdk.extensions.avro.coders.AvroCoder
 
 class PreProcessBigDiffyJobTest extends PipelineSpec {
   val lhs = Gen.listOfN(1000, ExampleAvroGen.exampleRecordGen).sample.get.map { r =>
     r.setNullableIntField(null)
     r
   }
 
-  val rhs = lhs.map(CoderUtils.clone(AvroCoder.of(classOf[ExampleRecord]), _)).map { r =>
+  val rhs = lhs.map(CoderUtils.clone(AvroCoder.reflect(classOf[ExampleRecord]), _)).map { r =>
     r.setNullableIntField(0)
     r
   }

ratatool-scalacheck/src/main/scala/com/spotify/ratatool/scalacheck/AvroGenerator.scala

@@ -23,7 +23,7 @@ import org.apache.avro._
 import org.apache.avro.generic.{GenericData, GenericRecord}
 import org.apache.avro.specific.SpecificRecord
 import org.apache.avro.util.Utf8
-import org.apache.beam.sdk.coders.{AvroCoder, AvroGenericCoder}
+import org.apache.beam.sdk.extensions.avro.coders.{AvroCoder, AvroGenericCoder}
 import org.apache.beam.sdk.util.CoderUtils
 import org.scalacheck.{Arbitrary, Gen}
 
@@ -52,7 +52,7 @@ object AvroGeneratorOps extends AvroGeneratorOps
 trait AvroGeneratorOps {
   def specificRecordOf[A <: SpecificRecord: ClassTag]: Gen[A] = {
     val cls = implicitly[ClassTag[A]].runtimeClass.asInstanceOf[Class[A]]
-    val specificCoder = AvroCoder.of(cls)
+    val specificCoder = AvroCoder.reflect(cls)
     val genericCoder = AvroGenericCoder.of(specificCoder.getSchema)
 
     genericRecordOf(specificCoder.getSchema).map { generic =>

ratatool-scalacheck/src/test/scala/com/spotify/ratatool/scalacheck/AvroGeneratorTest.scala

@@ -18,14 +18,14 @@
 package com.spotify.ratatool.scalacheck
 
 import com.spotify.ratatool.avro.specific.{RequiredNestedRecord, TestRecord}
-import org.apache.beam.sdk.coders.AvroCoder
+import org.apache.beam.sdk.extensions.avro.coders.AvroCoder
 import org.apache.beam.sdk.util.CoderUtils
 import org.scalacheck._
 import org.scalacheck.Prop.{all, forAll, propBoolean, AnyOperators}
 
 object AvroGeneratorTest extends Properties("AvroGenerator") {
   property("round trips") = forAll(specificRecordOf[TestRecord]) { m =>
-    val coder = AvroCoder.of(classOf[TestRecord])
+    val coder = AvroCoder.reflect(classOf[TestRecord])
 
     val bytes = CoderUtils.encodeToByteArray(coder, m)
     val decoded = CoderUtils.decodeFromByteArray(coder, bytes)