initial commit

MaximRouiller · MaximRouiller · commit bca192a72d60 · 2018-09-12T11:32:38.000-04:00
diff --git a/.gitignore b/.gitignore
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,4 @@
-## [project-title] Changelog
+## durablefunctions-apiscraping-nodejs Changelog
 
 <a name="x.y.z"></a>
 # x.y.z (yyyy-mm-dd)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -1,4 +1,4 @@
-# Contributing to [project-title]
+# Contributing to durablefunctions-apiscraping-nodejs
 
 This project welcomes contributions and suggestions.  Most contributions require you to agree to a
 Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
diff --git a/FanOutFanInCrawler/.gitignore b/FanOutFanInCrawler/.gitignore
@@ -0,0 +1,25 @@
+bin
+obj
+csx
+.vs
+edge
+Publish
+
+*.user
+*.suo
+*.cscfg
+*.Cache
+project.lock.json
+
+/packages
+/TestResults
+
+/tools/NuGet.exe
+/App_Data
+/secrets
+/data
+.secrets
+appsettings.json
+local.settings.json
+
+node_modules
diff --git a/FanOutFanInCrawler/.vscode/extensions.json b/FanOutFanInCrawler/.vscode/extensions.json
@@ -0,0 +1,5 @@
+{
+  "recommendations": [
+    "ms-azuretools.vscode-azurefunctions"
+  ]
+}
diff --git a/FanOutFanInCrawler/.vscode/launch.json b/FanOutFanInCrawler/.vscode/launch.json
@@ -0,0 +1,13 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Attach to JavaScript Functions",
+      "type": "node",
+      "request": "attach",
+      "port": 5858,
+      "protocol": "inspector",
+      "preLaunchTask": "runFunctionsHost"
+    }
+  ]
+}
diff --git a/FanOutFanInCrawler/.vscode/settings.json b/FanOutFanInCrawler/.vscode/settings.json
@@ -0,0 +1,5 @@
+{
+  "azureFunctions.projectRuntime": "beta",
+  "azureFunctions.projectLanguage": "JavaScript",
+  "azureFunctions.templateFilter": "Verified"
+}
diff --git a/FanOutFanInCrawler/.vscode/tasks.json b/FanOutFanInCrawler/.vscode/tasks.json
@@ -0,0 +1,38 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "Run Functions Host",
+      "identifier": "runFunctionsHost",
+      "type": "shell",
+      "command": "func host start",
+      "options": {
+        "env": {
+          "languageWorkers:node:arguments": "--inspect=5858"
+        }
+      },
+      "isBackground": true,
+      "presentation": {
+        "reveal": "always"
+      },
+      "problemMatcher": [
+        {
+          "owner": "azureFunctions",
+          "pattern": [
+            {
+              "regexp": "\\b\\B",
+              "file": 1,
+              "location": 2,
+              "message": 3
+            }
+          ],
+          "background": {
+            "activeOnStart": true,
+            "beginsPattern": "^.*Stopping host.*",
+            "endsPattern": "^.*Job host started.*"
+          }
+        }
+      ]
+    }
+  ]
+}
diff --git a/FanOutFanInCrawler/GetAllRepositoriesForOrganization/function.json b/FanOutFanInCrawler/GetAllRepositoriesForOrganization/function.json
@@ -0,0 +1,8 @@
+{
+  "bindings": [{
+    "name": "input",
+    "type": "activityTrigger",
+    "direction": "in"
+  }],
+  "disabled": false
+}
diff --git a/FanOutFanInCrawler/GetAllRepositoriesForOrganization/index.js b/FanOutFanInCrawler/GetAllRepositoriesForOrganization/index.js
@@ -0,0 +1,30 @@
+const octokit = require('@octokit/rest')({
+    headers: {
+        'user-agent': 'FanOutFanInCrawler'
+    }
+});
+octokit.authenticate({
+    type: 'token',
+    token: process.env["GitHubToken"]
+});
+
+module.exports = async function (context, input) {
+    // retrieves the organization name from the Orchestrator function
+    var organizationName = input;
+
+    var finalResult = [];
+    let page = 1;
+    do {
+        // invoke the API to retrieve the list of repositories of a specific organization
+        var result = await octokit.repos.getForOrg({
+            org: organizationName,
+            type: "public",
+            page: page
+        });
+        page++;
+        // merge the paged results inside a single array
+        finalResult = finalResult.concat(result.data);
+    }
+    while (result.data.length !== 0)
+    return finalResult;
+};
diff --git a/FanOutFanInCrawler/GetOpenedIssues/function.json b/FanOutFanInCrawler/GetOpenedIssues/function.json
@@ -0,0 +1,8 @@
+{
+  "bindings": [{
+    "name": "input",
+    "type": "activityTrigger",
+    "direction": "in"
+  }],
+  "disabled": false
+}
diff --git a/FanOutFanInCrawler/GetOpenedIssues/index.js b/FanOutFanInCrawler/GetOpenedIssues/index.js
@@ -0,0 +1,35 @@
+const df = require("durable-functions");
+const octokit = require('@octokit/rest')({
+    headers: {
+        'user-agent': 'FanOutFanInCrawler'
+    }
+});
+octokit.authenticate({
+    type: 'token',
+    token: process.env["GitHubToken"]
+});
+
+module.exports = async function (context, input) {
+    // `input` here is retrieved from the Orchestrator function `callActivityAsync` input parameter
+
+    let page = 1
+    let issueCount = 0;
+
+    do {
+        // retrieves a list of open issues from a specific repository
+        var result = await octokit.issues.getForRepo({
+            owner: input.owner.login,
+            repo: input.name,
+            state: 'open',
+            per_page: 100,
+            page: page
+        });
+        page++;
+        // aggregate the result
+        issueCount += result.data.length;
+    }
+    while (result.data.length !== 0)
+
+    // returns an object with the count of opened issues for a specific repository
+    return {id: input.id, openedIssues: issueCount, name: input.name };
+};
diff --git a/FanOutFanInCrawler/Orchestrator/function.json b/FanOutFanInCrawler/Orchestrator/function.json
@@ -0,0 +1,8 @@
+{
+  "bindings": [{
+    "name": "context",
+    "type": "orchestrationTrigger",
+    "direction": "in"
+  }],
+  "disabled": false
+}
diff --git a/FanOutFanInCrawler/Orchestrator/index.js b/FanOutFanInCrawler/Orchestrator/index.js
@@ -0,0 +1,24 @@
+const df = require("durable-functions");
+
+module.exports = df(function* (context) {
+    // retrieves the organization name from the Orchestrator_HttpStart function
+    var organizationName = context.df.getInput();
+    // retrieves the list of repositories for an organization by invoking a separate Activity Function.
+    var repositories = yield context.df.callActivityAsync("GetAllRepositoriesForOrganization", organizationName);
+
+    // Creates an array of task to store the result of each functions
+    var output = [];
+    for (var i = 0; i < repositories.length; i++) {
+        // Starting a `GetOpenedIssues` activity WITHOUT `yield`
+        // This will starts Activity Functions in parallel instead of sequentially.
+        output.push(context.df.callActivityAsync("GetOpenedIssues", repositories[i]));
+    }
+
+    // Wait for all Activity Functions to complete execution
+    const results = yield context.df.Task.all(output);
+
+    // Send the list to an Activity Function to save them to Blob Storage.
+    yield context.df.callActivityAsync("SaveRepositories", results);
+
+    return context.instanceId;
+});
diff --git a/FanOutFanInCrawler/Orchestrator_HttpStart/function.json b/FanOutFanInCrawler/Orchestrator_HttpStart/function.json
@@ -0,0 +1,25 @@
+{
+  "disabled": false,
+  "bindings": [
+    {
+      "authLevel": "function",
+      "type": "httpTrigger",
+      "direction": "in",
+      "name": "req",
+      "methods": [
+        "get",
+        "post"
+      ]
+    },
+    {
+      "type": "http",
+      "direction": "out",
+      "name": "res"
+    },
+    {
+      "name": "starter",
+      "type": "orchestrationClient",
+      "direction": "out"
+    }
+  ]
+}
diff --git a/FanOutFanInCrawler/Orchestrator_HttpStart/index.js b/FanOutFanInCrawler/Orchestrator_HttpStart/index.js
@@ -0,0 +1,14 @@
+const uuid = require('uuid/v4');
+
+module.exports = function (context, req, starter) {
+    var id = uuid();
+    
+    // Function input comes from the request content.
+    context.bindings.starter = [{
+        FunctionName: "Orchestrator",
+        Input: "Nuget",
+        InstanceId: id
+    }];
+    
+    context.done(null);
+};
diff --git a/FanOutFanInCrawler/SaveRepositories/function.json b/FanOutFanInCrawler/SaveRepositories/function.json
@@ -0,0 +1,8 @@
+{
+  "bindings": [{
+    "name": "input",
+    "type": "activityTrigger",
+    "direction": "in"
+  }],
+  "disabled": false
+}
diff --git a/FanOutFanInCrawler/SaveRepositories/index.js b/FanOutFanInCrawler/SaveRepositories/index.js
@@ -0,0 +1,37 @@
+const df = require("durable-functions");
+var storage = require('azure-storage');
+
+module.exports = async function (context, input) {
+    // `input` here is retrieved from the Orchestrator function `callActivityAsync` input parameter
+
+    // create the table service for Blob Storage
+    var tableService = storage.createTableService(process.env['AzureWebJobsStorage']);
+    
+    // create the table if it doesn't exist already.
+    tableService.createTableIfNotExists('Repositories', function (error) {
+        if (error) {
+            console.error(error);
+        }
+        if (!error) {
+            // creates a batch of operation to be executed
+            var batch = new storage.TableBatch();
+            for (var i = 0; i < input.length; i++) {
+                var repository = input[i];
+
+                // Creates an operation to add the repository to Table Storage
+                batch.insertOrReplaceEntity({
+                    PartitionKey: {'_': 'Default'},
+                    RowKey: {'_': repository.id.toString()},
+                    OpenedIssues: {'_': repository.openedIssues},
+                    RepositoryName: {'_': repository.name}
+                });
+            }
+            // execute the batch of operations
+            tableService.executeBatch('Repositories', batch, function (error, result, response) {
+                if (error) {
+                    console.error(error);
+                }
+            });
+        }
+    });
+};
diff --git a/FanOutFanInCrawler/extensions.csproj b/FanOutFanInCrawler/extensions.csproj
@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+	<WarningsAsErrors></WarningsAsErrors>
+	<DefaultItemExcludes>**</DefaultItemExcludes>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="1.6.0" />
+    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Http" Version="3.0.0-beta8" />
+    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="3.0.0-beta8" />
+    <PackageReference Include="Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator" Version="1.0.0" />
+  </ItemGroup>
+</Project>
diff --git a/FanOutFanInCrawler/host.json b/FanOutFanInCrawler/host.json
diff --git a/FanOutFanInCrawler/jsconfig.json b/FanOutFanInCrawler/jsconfig.json
diff --git a/FanOutFanInCrawler/local.settings.json b/FanOutFanInCrawler/local.settings.json
diff --git a/FanOutFanInCrawler/package-lock.json b/FanOutFanInCrawler/package-lock.json
diff --git a/FanOutFanInCrawler/package.json b/FanOutFanInCrawler/package.json
diff --git a/README.md b/README.md

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-## [project-title] Changelog`
	`1`	`+## durablefunctions-apiscraping-nodejs Changelog`
`2`	`2`
`3`	`3`	`<a name="x.y.z"></a>`
`4`	`4`	`# x.y.z (yyyy-mm-dd)`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-# Contributing to [project-title]`
	`1`	`+# Contributing to durablefunctions-apiscraping-nodejs`
`2`	`2`
`3`	`3`	`This project welcomes contributions and suggestions. Most contributions require you to agree to a`
`4`	`4`	`Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +{
 +  "recommendations": [
 +    "ms-azuretools.vscode-azurefunctions"
 +  ]
 +}