Skip to content

Commit 9193dd0

Browse files
evantahlerevantahler
authored
Generate connectors.md (#23367)
* Generate `connectors.md` * fixed image size * small code icons * smaller icons * ignore PMD warnings * add link to code and fix !=
1 parent 2c20bf0 commit 9193dd0

File tree

3 files changed

+464
-1
lines changed

3 files changed

+464
-1
lines changed

‎airbyte-config/specs/build.gradle

+19-1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,24 @@ task generateOssConnectorCatalog(type: JavaExec, dependsOn: generateSeedConnecto
4545
args 'oss_catalog.json'
4646
}
4747

48+
project(":airbyte-config:init").tasks.processResources.dependsOn(generateOssConnectorCatalog)
49+
50+
task generateConnectorMarkdown(type: JavaExec, dependsOn: generateSeedConnectorSpecs) {
51+
classpath = sourceSets.main.runtimeClasspath
52+
53+
mainClass = 'io.airbyte.config.specs.ConnectorMarkdownGenerator'
54+
55+
args '--project-root'
56+
args new File(project(":").projectDir, "")
57+
58+
args '--seed-root'
59+
args new File(project(":airbyte-config:init").projectDir, '/src/main/resources/seed')
60+
61+
args '--output-filename'
62+
args 'connectors.md'
63+
}
64+
65+
project(":airbyte-config:init").tasks.processResources.dependsOn(generateConnectorMarkdown)
66+
4867
Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)
4968

50-
project(":airbyte-config:init").tasks.processResources.dependsOn(generateOssConnectorCatalog)

‎airbyte-config/specs/src/main/java/io/airbyte/config/specs/ConnectorMarkdownGenerator.java

+111
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
package io.airbyte.config.specs;
2+
3+
import com.fasterxml.jackson.databind.JsonNode;
4+
import io.airbyte.commons.cli.Clis;
5+
import io.airbyte.commons.io.IOs;
6+
import io.airbyte.commons.util.MoreIterators;
7+
import io.airbyte.commons.yaml.Yamls;
8+
import java.nio.file.Path;
9+
import java.time.LocalDateTime;
10+
import java.time.format.DateTimeFormatter;
11+
import java.util.ArrayList;
12+
import java.util.Comparator;
13+
import java.util.List;
14+
import org.apache.commons.cli.CommandLine;
15+
import org.apache.commons.cli.Option;
16+
import org.apache.commons.cli.Options;
17+
18+
public class ConnectorMarkdownGenerator {
19+
20+
private static final Option PROJECT_ROOT_OPTION = Option.builder("p").longOpt("project-root").hasArg(true).required(true)
21+
.desc("path to where seed resource files are stored").build();
22+
23+
private static final Option SEED_ROOT_OPTION = Option.builder("s").longOpt("seed-root").hasArg(true).required(true)
24+
.desc("path to where seed resource files are stored").build();
25+
private static final Option OUTPUT_FILENAME_OPTION = Option.builder("o").longOpt("output-filename").hasArg(true).required(true)
26+
.desc("name for the generated catalog json file").build();
27+
private static final Options OPTIONS = new Options().addOption(PROJECT_ROOT_OPTION).addOption(SEED_ROOT_OPTION).addOption(OUTPUT_FILENAME_OPTION);
28+
29+
private static final String githubCodeBase = "https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors";
30+
private static final String githubIconBase = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons";
31+
private static final String iconSize = "30";
32+
33+
DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd @ HH:mm:ss");
34+
public static void main(final String[] args) throws Exception {
35+
final CommandLine parsed = Clis.parse(args, OPTIONS);
36+
final Path outputRoot = Path.of(parsed.getOptionValue(PROJECT_ROOT_OPTION.getOpt()));
37+
final Path seedRoot = Path.of(parsed.getOptionValue(SEED_ROOT_OPTION.getOpt()));
38+
final String outputFileName = parsed.getOptionValue(OUTPUT_FILENAME_OPTION.getOpt());
39+
40+
final ConnectorMarkdownGenerator mdGenerator = new ConnectorMarkdownGenerator();
41+
mdGenerator.run(outputRoot, seedRoot, outputFileName);
42+
}
43+
44+
public void run(final Path outputRoot, final Path seedRoot, final String outputFileName) {
45+
final List<JsonNode> destinationDefinitionsJson = getSeedJson(seedRoot, SeedConnectorType.DESTINATION.getDefinitionFileName());
46+
final List<JsonNode> sourceDefinitionsJson = getSeedJson(seedRoot, SeedConnectorType.SOURCE.getDefinitionFileName());
47+
String body = buildMarkdown(sourceDefinitionsJson, destinationDefinitionsJson);
48+
IOs.writeFile(outputRoot.resolve(outputFileName), body);
49+
}
50+
51+
private List<JsonNode> getSeedJson(final Path root, final String fileName) {
52+
final String jsonString = IOs.readFile(root, fileName);
53+
return MoreIterators.toList(Yamls.deserialize(jsonString).elements())
54+
.stream()
55+
.sorted(Comparator.comparing(o -> o.get("name").asText()))
56+
.toList();
57+
}
58+
59+
private String buildMarkdown (List<JsonNode> sourceDefinitionsJson, List<JsonNode> destinationDefinitionsJson) {
60+
final List<String> bodyParts = new ArrayList<>();
61+
62+
bodyParts.add("# Airbyte Connectors");
63+
bodyParts.add("__Generated: " + dtf.format(LocalDateTime.now()) + "__");
64+
65+
bodyParts.add("");
66+
bodyParts.add("## Sources");
67+
bodyParts.add("");
68+
bodyParts.add(buildMarkdownTable(sourceDefinitionsJson, "Source"));
69+
70+
bodyParts.add("");
71+
bodyParts.add("## Destinations");
72+
bodyParts.add("");
73+
bodyParts.add(buildMarkdownTable(destinationDefinitionsJson, "Destination"));
74+
75+
return String.join("\r\n", bodyParts);
76+
}
77+
78+
@SuppressWarnings("PMD")
79+
private String buildMarkdownTable (List<JsonNode> definitions, String type) {
80+
final List<String> bodyParts = new ArrayList<>();
81+
82+
List<String> headers = new ArrayList<>();
83+
headers.add("Name");
84+
headers.add("Icon");
85+
headers.add("Type");
86+
headers.add("Image");
87+
headers.add("Release Stage");
88+
headers.add("Docs");
89+
headers.add("Code");
90+
headers.add("ID");
91+
92+
bodyParts.add("| " + String.join(" | ", headers) + " |");
93+
bodyParts.add("|----|----|----|----|----|----|----|----|");
94+
for (final JsonNode definition : definitions) {
95+
final String name = definition.get("name").asText();
96+
final String codeName = definition.get("dockerRepository").asText().split("/")[1];
97+
final String icon = definition.get("icon") != null ? definition.get("icon").asText() : "";
98+
final String iconLink = !icon.equals("") ? "<img alt=\"" + name + " icon\" src=\"" + githubIconBase + "/" + icon + "\" height=\"" + iconSize + "\" height=\"" + iconSize + "\"/>" : "x";
99+
final String dockerImage = definition.get("dockerRepository").asText() + ":" + definition.get("dockerImageTag").asText();
100+
final String releaseStage = definition.get("releaseStage") != null ? definition.get("releaseStage").asText() : "unknown";
101+
final String documentationUrl = definition.get("documentationUrl") != null ? definition.get("documentationUrl").asText() : "";
102+
final String docLink = !documentationUrl.equals("") ? "[link](" + documentationUrl + ")" : "missing";
103+
final String codeLink = "[code](" + githubCodeBase + "/" + codeName + ")";
104+
final String id = "<small>`" + definition.get(type.toLowerCase() + "DefinitionId").asText() + "`</small>";
105+
106+
bodyParts.add("| **" + name + "** | " + iconLink + " | " + type + " | " + dockerImage + " | " + releaseStage + " | " + docLink + " | " + codeLink + " | " + id + " |");
107+
}
108+
109+
return String.join("\r\n", bodyParts);
110+
}
111+
}

0 commit comments

Comments
 (0)