Merge pull request tensorflow#2885 from ddhruvkr:add_wiki_auto

PiperOrigin-RevId: 417765733

tensorflow_datasets/text_simplification/__init__.py

@@ -16,3 +16,4 @@
 """Text simplification datasets."""
 
 from tensorflow_datasets.text_simplification.asset import Asset
+from tensorflow_datasets.text_simplification.wiki_auto import WikiAuto

tensorflow_datasets/text_simplification/wiki_auto/__init__.py

@@ -0,0 +1,18 @@
+# coding=utf-8
+# Copyright 2021 The TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""wiki_auto dataset."""
+
+from tensorflow_datasets.text_simplification.wiki_auto.wiki_auto import WikiAuto

tensorflow_datasets/text_simplification/wiki_auto/checksums.tsv

@@ -0,0 +1,15 @@
+https://dl.dropboxusercontent.com/sh/ohqaw41v48c7e5p/AAATBDhU1zpdcT5x5WgO8DMaa/wiki-auto-all-data/wiki-auto-part-1-data.json	2067424750	136d8e113a773d3669228a57cae733fca079954daf0b3514505410c66d1a69b6	wiki-auto-part-1-data.json
+https://dl.dropboxusercontent.com/sh/ohqaw41v48c7e5p/AAATgPkjo_tPt9z12vZxJ3MRa/wiki-auto-all-data/wiki-auto-part-2-data.json	93214171	94b33a11447c121a0ce7293de20fb969c36d8a62b31afc5873a4174ed17a1d4e	wiki-auto-part-2-data.json
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-auto/ACL2020/train.dst	47859304	d9e2106722e2e29f34d5d9b697c236b38e920724727cefb71f42072dd9fd8807	train.dst
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-auto/ACL2020/train.src	70209062	02141edbb735be50c9942f5e0bced4528dc8d844753d46a1f3bdf0b6e550c0e6	train.src
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-auto/GEM2021/full_no_split/train.dst	55545281	37b373c5ee8c3f4b844c9896f07fb6bb55a20a3d4a8e1a6b06a651b87a653415	train.dst
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-auto/GEM2021/full_no_split/train.src	86028898	48d820fc1e7bd7121edde04a2aedf62bb9069b9caed01b77b7cb3852cc6b33d0	train.src
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-auto/GEM2021/full_with_split/train.dst	53288474	74529cfc316c8a3ed7720f87e25d0758620e0e838a5747ec4a1bbdccef018f75	train.dst
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-auto/GEM2021/full_with_split/train.src	67389841	f70dd6083bf679254dfd6c1e14960ad2b526bc35596d9c87b0c636c0285ce8ac	train.src
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-auto/all_data/wiki-auto-part-1-data.json	2067424750	136d8e113a773d3669228a57cae733fca079954daf0b3514505410c66d1a69b6	wiki-auto-part-1-data.json
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-auto/all_data/wiki-auto-part-2-data.json	93214171	94b33a11447c121a0ce7293de20fb969c36d8a62b31afc5873a4174ed17a1d4e	wiki-auto-part-2-data.json
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-manual/dev.tsv	21534366	8625b81baa7f0b0060f00781974633458016a81352a15b8398f6740a763b4cd8	dev.tsv
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-manual/test.tsv	34530535	fd427cd0d2c2f268d12c15108041822f6b41c4041d07145ddf5711e03367dfba	test.tsv
+https://github.com/chaojiang06/wiki-auto/raw/master/wiki-manual/train.tsv	106346588	82fa388de3ded6d303b95fcd11ba70e0b6158d2df1cbf24913bb54503bd32e95	train.tsv
+https://www.dropbox.com/sh/ohqaw41v48c7e5p/AAATBDhU1zpdcT5x5WgO8DMaa/wiki-auto-all-data/wiki-auto-part-1-data.json	408939	3765bc7c309ecd2500f2b86344bed0a1e98c32b87bd759aacdb1a4696ca136fa	wiki-auto-part-1-data.json
+https://www.dropbox.com/sh/ohqaw41v48c7e5p/AAATBDhU1zpdcT5x5WgO8DMaa/wiki-auto-all-data/wiki-auto-part-2-data.json	143850	896777450c3db21d0edf5bd6d6df1d7b7f0db1e628e6da58953d02bb87ea78af	wiki-auto-part-2-data.json

tensorflow_datasets/text_simplification/wiki_auto/dummy_data/auto/1.0.0/wiki-auto-part-1-data.json

@@ -0,0 +1 @@
+{"0": {"simple": {"id": 702227, "title": "Lata Mondal", "url": "https://simple.wikipedia.org/wiki?curid=702227", "content": {"simple-702227-0-0": "Lata Mondal (born: 16 January 1993) is a Bangladeshi cricketer who plays for the Bangladesh national women's cricket team.", "simple-702227-0-1": "She is a right handed bat."}}, "normal": {"id": 41918715, "title": "Lata Mondal", "url": "https://en.wikipedia.org/wiki?curid=41918715", "content": {"normal-41918715-0-0": "Lata Mondal ( ; born: 16 January 1993, Dhaka) is a Bangladeshi cricketer who plays for the Bangladesh national women's cricket team.", "normal-41918715-0-1": "She is a right handed batter.", "normal-41918715-1-0": "Mondal was born on January 16, 1993 in Dhaka, Bangladesh.", "normal-41918715-2-0": "Mondal made her ODI career against the Ireland women's cricket team on November 26, 2011.", "normal-41918715-3-0": "Mondal made her T20I career against the Ireland women's cricket team on August 28, 2012.", "normal-41918715-3-1": "In October 2018, she was named in Bangladesh's squad for the 2018 ICC Women's World Twenty20 tournament in the West Indies.", "normal-41918715-4-0": "Mondal was a member of the team that won a silver medal in cricket against the China national women's cricket team at the 2010 Asian Games in Guangzhou, China."}}, "sentence_alignment": [["simple-702227-0-0", "normal-41918715-0-0"], ["simple-702227-0-1", "normal-41918715-0-1"]], "paragraph_alignment": [["simple-702227-0", "normal-41918715-0"]]}}

tensorflow_datasets/text_simplification/wiki_auto/dummy_data/auto/1.0.0/wiki-auto-part-2-data.json

@@ -0,0 +1 @@
+{"0": {"simple": {"id": 78856, "title": "Vex, Switzerland", "url": "https://simple.wikipedia.org/wiki?curid=78856", "content": {"simple-78856-0-0": "Vex is the capital city of the district of He\u0301rens in the canton of Valais in Switzerland."}}, "normal": {"id": 7023314, "title": "Vex, Switzerland", "url": "https://en.wikipedia.org/wiki?curid=7023314", "content": {"normal-7023314-0-0": "Vex is a municipality and capital of the district of He\u0301rens in the canton of Valais in Switzerland.", "normal-7023314-1-0": "Vex has an area, as of 2009 , of 13 km2 .", "normal-7023314-1-1": "Of this area, 4.17 km2 or 32.1% is used for agricultural purposes, while 6.17 km2 or 47.5% is forested.", "normal-7023314-1-2": "Of the rest of the land, 1.71 km2 or 13.2% is settled (buildings or roads), 0.07 km2 or 0.5% is either rivers or lakes and 0.87 km2 or 6.7% is unproductive land.", "normal-7023314-2-0": "Of the built up area, housing and buildings made up 7.2% and transportation infrastructure made up 4.8%.", "normal-7023314-2-1": "Out of the forested land, 38.6% of the total land area is heavily forested and 7.7% is covered with orchards or small clusters of trees.", "normal-7023314-2-2": "Of the agricultural land, 0.2% is used for growing crops and 18.3% is pastures, while 2.5% is used for orchards or vine crops and 11.0% is used for alpine pastures.", "normal-7023314-2-3": "All the water in the municipality is flowing water.", "normal-7023314-2-4": "Of the unproductive areas, 5.7% is unproductive vegetation.", "normal-7023314-3-0": "The blazon of the municipal coat of arms is \"Azure, a Chevronelle inverted couped Argent between three Mullets of Five of the same and Coupeaux Vert in base.\"", "normal-7023314-4-0": "Vex has a population (as of December 2017 ) of .", "normal-7023314-4-1": "s of 2008 , 11.3% of the population are resident foreign nationals.", "normal-7023314-4-2": "Over the last 10 years (1999\u20132009 ) the population has changed at a rate of 23.8%.", "normal-7023314-4-3": "It has changed at a rate of 23% due to migration and at a rate of -0.7% due to births and deaths.", "normal-7023314-5-0": "Most of the population (as of 2000 ) speaks French (1,216 or 92.8%) as their first language, German is the second most common (38 or 2.9%) and Portuguese is the third (30 or 2.3%).", "normal-7023314-5-1": "There are 4 people who speak Italian.", "normal-7023314-6-0": "s of 2008 , the gender distribution of the population was 50.7% male and 49.3% female.", "normal-7023314-6-1": "The population was made up of 724 Swiss men (44.4% of the population) and 103 (6.3%) non-Swiss men.", "normal-7023314-6-2": "There were 721 Swiss women (44.2%) and 83 (5.1%) non-Swiss women.", "normal-7023314-6-3": "Of the population in the municipality 535 or about 40.8% were born in Vex and lived there in 2000.", "normal-7023314-6-4": "There were 350 or 26.7% who were born in the same canton, while 157 or 12.0% were born somewhere else in Switzerland, and 173 or 13.2% were born outside of Switzerland.", "normal-7023314-7-0": "The age distribution of the population (as of 2000 ) is children and teenagers (0\u201319 years old) make up 24.1% of the population, while adults (20\u201364 years old) make up 56.9% and seniors (over 64 years old) make up 19%.", "normal-7023314-8-0": "s of 2000 , there were 526 people who were single and never married in the municipality.", "normal-7023314-8-1": "There were 604 married individuals, 126 widows or widowers and 55 individuals who are divorced.", "normal-7023314-9-0": "s of 2000 , there were 514 private households in the municipality, and an average of 2.3 persons per household.", "normal-7023314-9-1": "There were 180 households that consist of only one person and 28 households with five or more people.", "normal-7023314-9-2": "Out of a total of 528 households that answered this question, 34.1% were households made up of just one person and there were 6 adults who lived with their parents.", "normal-7023314-9-3": "Of the rest of the households, there are 129 married couples without children, 169 married couples with children There were 24 single parents with a child or children.", "normal-7023314-9-4": "There were 6 households that were made up of unrelated people and 14 households that were made up of some sort of institution or another collective housing.", "normal-7023314-10-0": "In 2000 there were 626 single family homes (or 72.3% of the total) out of a total of 866 inhabited buildings.", "normal-7023314-10-1": "There were 135 multi-family buildings (15.6%), along with 51 multi-purpose buildings that were mostly used for housing (5.9%) and 54 other use buildings (commercial or industrial) that also had some housing (6.2%).", "normal-7023314-11-0": "In 2000 , a total of 498 apartments (28.4% of the total) were permanently occupied, while 1,186 apartments (67.7%) were seasonally occupied and 69 apartments (3.9%) were empty.", "normal-7023314-11-1": "s of 2009 , the construction rate of new housing units was 15.3 new units per 1000 residents.", "normal-7023314-12-0": "The historical population is given in the following chart: In the 2007 federal election the most popular party was the CVP which received 37.97% of the vote.", "normal-7023314-12-1": "The next three most popular parties were the FDP (21.97%), the SVP (19.71%) and the SP (12.57%).", "normal-7023314-12-2": "In the federal election, a total of 724 votes were cast, and the voter turnout was 68.6%.", "normal-7023314-13-0": "In the 2009 Conseil d'Etat/Staatsrat election a total of 676 votes were cast, of which 41 or about 6.1% were invalid.", "normal-7023314-13-1": "The voter participation was 63.3%, which is much more than the cantonal average of 54.67%.", "normal-7023314-13-2": "In the 2007 Swiss Council of States election a total of 718 votes were cast, of which 52 or about 7.2% were invalid.", "normal-7023314-13-3": "The voter participation was 69.4%, which is much more than the cantonal average of 59.88%.", "normal-7023314-14-0": "s of 2010 , Vex had an unemployment rate of 2.8%.", "normal-7023314-14-1": "s of 2008 , there were 50 people employed in the primary economic sector and about 18 businesses involved in this sector.", "normal-7023314-14-2": "58 people were employed in the secondary sector and there were 11 businesses in this sector.", "normal-7023314-14-3": "309 people were employed in the tertiary sector, with 57 businesses in this sector.", "normal-7023314-14-4": "There were 644 residents of the municipality who were employed in some capacity, of which females made up 43.3% of the workforce.", "normal-7023314-15-0": "In 2008 the total number of full-time equivalent jobs was 331.", "normal-7023314-15-1": "The number of jobs in the primary sector was 34, of which 20 were in agriculture and 14 were in forestry or lumber production.", "normal-7023314-15-2": "The number of jobs in the secondary sector was 54 of which 1 was in manufacturing and 53 (98.1%) were in construction.", "normal-7023314-15-3": "The number of jobs in the tertiary sector was 243.", "normal-7023314-15-4": "In the tertiary sector; 42 or 17.3% were in wholesale or retail sales or the repair of motor vehicles, 16 or 6.6% were in the movement and storage of goods, 55 or 22.6% were in a hotel or restaurant, 13 or 5.3% were the insurance or financial industry, 6 or 2.5% were technical professionals or scientists, 10 or 4.1% were in education and 53 or 21.8% were in health care.", "normal-7023314-16-0": "In 2000 , there were 153 workers who commuted into the municipality and 406 workers who commuted away.", "normal-7023314-16-1": "The municipality is a net exporter of workers, with about 2.7 workers leaving the municipality for every one entering.", "normal-7023314-16-2": "Of the working population, 8.9% used public transportation to get to work, and 71.4% used a private car.", "normal-7023314-17-0": "From the 2000 census , 1,047 or 79.9% were Roman Catholic, while 78 or 5.9% belonged to the Swiss Reformed Church.", "normal-7023314-17-1": "Of the rest of the population, there were 5 members of an Orthodox church (or about 0.38% of the population), there was 1 individual who belongs to the Christian Catholic Church, and there was 1 individual who belongs to another Christian church.", "normal-7023314-17-2": "There was 1 individual who was Jewish, and 4 (or about 0.31% of the population) who were Islamic.", "normal-7023314-17-3": "There was 1 person who was Buddhist and 1 individual who belonged to another church.", "normal-7023314-17-4": "57 (or about 4.35% of the population) belonged to no church, are agnostic or atheist, and 115 individuals (or about 8.77% of the population) did not answer the question.", "normal-7023314-18-0": "In Vex about 448 or (34.2%) of the population have completed non-mandatory upper secondary education, and 125 or (9.5%) have completed additional higher education (either university or a \"Fachhochschule\").", "normal-7023314-18-1": "Of the 125 who completed tertiary schooling, 57.6% were Swiss men, 28.8% were Swiss women, 7.2% were non-Swiss men and 6.4% were non-Swiss women.", "normal-7023314-18-2": "s of 2000 , there were 25 students in Vex who came from another municipality, while 69 residents attended schools outside the municipality.", "normal-7023314-19-0": "Vex is home to the \"Bibliothe\u0300que de Vex\" library.", "normal-7023314-19-1": "The library has (as of 2008 ) 4,671 books or other media, and loaned out 6,911 items in the same year.", "normal-7023314-19-2": "It was open a total of 79 days with average of 6 hours per week during that year."}}, "sentence_alignment": []}}

tensorflow_datasets/text_simplification/wiki_auto/dummy_data/auto_acl/1.0.0/train.dst

@@ -0,0 +1,5 @@
+Pterocarpus indicus -LRB- commonly known as Amboyna wood , Malay padauk , Papua New Guinea rosewood , Philippine mahogany , Andaman redwood , Burmese rosewood , narra or Pashu padauk -RRB- is a species of " Pterocarpus " native to southeastern Asia .
+A romantic friendship , passionate friendship or affectionate friendship is a close but non-sexual relationship between friends that often involves a degree of physical and emotional closeness .
+Rocky Balboa -LRB- also known as Rocky VI -RRB- is the sixth and final movie in the Rocky series , written , directed by and starring Sylvester Stallone .
+Janakinath Bose -LRB- 28 May 1860 – 1934 -RRB- was an Indian lawyer and the father of Indian independence leader Subhas Chandra Bose .
+Bionics -LRB- also known as biomimetics -RRB- is the science about studying nature to create modern technology .

tensorflow_datasets/text_simplification/wiki_auto/dummy_data/auto_acl/1.0.0/train.src

@@ -0,0 +1,5 @@
+Pterocarpus indicus -LRB- commonly known as Amboyna wood , Malay padauk , Papua New Guinea rosewood , Philippine mahogany , Andaman redwood , Burmese rosewood , narra , angsana , or Pashu padauk -RRB- is a species of " Pterocarpus " native to southeastern Asia , northern Australasia , and the western Pacific Ocean islands , in Cambodia , southernmost China , East Timor , Indonesia , Malaysia , Papua New Guinea , the Philippines , the Ryukyu Islands , the Solomon Islands , Thailand , and Vietnam .
+A romantic friendship , passionate friendship , or affectionate friendship is a very close but typically non-sexual relationship between friends , often involving a degree of physical closeness beyond that which is common in the contemporary Western societies .
+Rocky Balboa is a 2006 American boxing sports drama film written , directed by , and starring Sylvester Stallone .
+Janakinath Bose -LRB- 28 May 1860 – 1934 -RRB- was an Indian lawyer and advocate , who was the father of Indian independence leader Subhas Chandra Bose .
+Bionics or biologically inspired engineering is the application of biological methods and systems found in nature to the study and design of engineering systems and modern technology .

tensorflow_datasets/text_simplification/wiki_auto/dummy_data/auto_full_no_split/1.0.0/train.dst

@@ -0,0 +1,5 @@
+The U.S. State of Texas was affected by at least 34 tropical or subtropical cyclones between 1950 and 1979 .
+Safe House is a 2012 American action thriller movie .
+The 1936 Olympic football tournament was won by Italy .
+He was a member of the Wedgwood family , who were very successful potters .
+Online comments describing the film usually described it as good and emotional .

tensorflow_datasets/text_simplification/wiki_auto/dummy_data/auto_full_no_split/1.0.0/train.src

@@ -0,0 +1,5 @@
+From 1950 to 1979 , at least 34 tropical or subtropical cyclones affected the U.S. state of Texas .
+Safe House is a 2012 South African & American action thriller film directed by Daniel Espinosa , starring Denzel Washington and Ryan Reynolds .
+Football at the 1936 Summer Olympics was won by Italy .
+Wedgwood was the son of the potter Thomas Wedgwood III and his wife Mary Stringer .
+Online reviews and comments universally praised the film as heartfelt and artistically vibrant , though it was criticized for its brevity .

tensorflow_datasets/text_simplification/wiki_auto/dummy_data/auto_full_with_split/1.0.0/train.dst

@@ -0,0 +1,5 @@
+Pterocarpus indicus ( commonly known as Amboyna wood , Malay padauk , Papua New Guinea rosewood , Philippine mahogany , Andaman redwood , Burmese rosewood , narra or Pashu padauk ) is a species of " Pterocarpus " native to southeastern Asia .
+A romantic friendship , passionate friendship or affectionate friendship is a close but non-sexual relationship between friends that often involves a degree of physical and emotional closeness .
+Rocky Balboa ( also known as Rocky VI ) is the sixth and final movie in the Rocky series , written , directed by and starring Sylvester Stallone .
+Janakinath Bose ( 28 May 1860 – 1934 ) was an Indian lawyer and the father of Indian independence leader Subhas Chandra Bose .
+Bionics ( also known as biomimetics ) is the science about studying nature to create modern technology .

tensorflow_datasets/text_simplification/wiki_auto/dummy_data/auto_full_with_split/1.0.0/train.src

@@ -0,0 +1,5 @@
+Pterocarpus indicus ( commonly known as Amboyna wood , Malay padauk , Papua New Guinea rosewood , Philippine mahogany , Andaman redwood , Burmese rosewood , narra , angsana , or Pashu padauk ) is a species of " Pterocarpus " native to southeastern Asia , northern Australasia , and the western Pacific Ocean islands , in Cambodia , southernmost China , East Timor , Indonesia , Malaysia , Papua New Guinea , the Philippines , the Ryukyu Islands , the Solomon Islands , Thailand , and Vietnam .
+A romantic friendship , passionate friendship , or affectionate friendship is a very close but typically non-sexual relationship between friends , often involving a degree of physical closeness beyond that which is common in the contemporary Western societies .
+Rocky Balboa is a 2006 American boxing sports drama film written , directed by , and starring Sylvester Stallone .
+Janakinath Bose ( 28 May 1860 – 1934 ) was an Indian lawyer and advocate , who was the father of Indian independence leader Subhas Chandra Bose .
+Bionics or biologically inspired engineering is the application of biological methods and systems found in nature to the study and design of engineering systems and modern technology .