Skip to content

Commit f3a0380

Browse files
committed
[INLONG-4128][Manager] Abstracting the logic for creating Hive tables
1 parent ad5e873 commit f3a0380

28 files changed

+455
-1185
lines changed
+15-3
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,22 @@
1515
* limitations under the License.
1616
*/
1717

18-
package org.apache.inlong.manager.common.pojo.query.hive;
18+
package org.apache.inlong.manager.common.enums;
1919

20-
import org.apache.inlong.manager.common.pojo.query.ColumnInfoBean;
20+
import java.util.Locale;
2121

22-
public class HiveColumnInfoBean extends ColumnInfoBean {
22+
public enum DataNodeType {
23+
24+
HIVE,
25+
KAFKA,
26+
ICEBERG,
27+
CLICKHOUSE,
28+
ES,
29+
;
30+
31+
@Override
32+
public String toString() {
33+
return this.name().toUpperCase(Locale.ROOT);
34+
}
2335

2436
}

inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/query/ColumnInfoBean.java

-29
This file was deleted.

inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/query/ColumnQueryBean.java

-49
This file was deleted.

inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/query/ConnectionInfo.java

-40
This file was deleted.

inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/query/DatabaseDetail.java

-41
This file was deleted.

inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/query/DatabaseQueryBean.java

-43
This file was deleted.

inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/query/TableQueryBean.java

-56
This file was deleted.

inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/query/hive/HiveColumnQueryBean.java

-37
This file was deleted.

inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/sink/hive/HiveSinkDTO.java

+33
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import lombok.Builder;
2525
import lombok.Data;
2626
import lombok.NoArgsConstructor;
27+
import org.apache.commons.collections.CollectionUtils;
2728
import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
2829
import org.apache.inlong.manager.common.exceptions.BusinessException;
2930

@@ -110,6 +111,9 @@ public static HiveSinkDTO getFromRequest(HiveSinkRequest request) {
110111
.build();
111112
}
112113

114+
/**
115+
* Get Hive sink info from JSON string
116+
*/
113117
public static HiveSinkDTO getFromJson(@NotNull String extParams) {
114118
try {
115119
OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
@@ -119,4 +123,33 @@ public static HiveSinkDTO getFromJson(@NotNull String extParams) {
119123
}
120124
}
121125

126+
/**
127+
* Get Hive table info
128+
*/
129+
public static HiveTableInfo getHiveTableInfo(HiveSinkDTO hiveInfo, List<HiveColumnInfo> columnList) {
130+
HiveTableInfo tableInfo = new HiveTableInfo();
131+
tableInfo.setDbName(hiveInfo.getDbName());
132+
tableInfo.setTableName(hiveInfo.getTableName());
133+
134+
// Set partition fields
135+
if (CollectionUtils.isNotEmpty(hiveInfo.getPartitionFieldList())) {
136+
for (HivePartitionField field : hiveInfo.getPartitionFieldList()) {
137+
HiveColumnInfo columnInfo = new HiveColumnInfo();
138+
columnInfo.setName(field.getFieldName());
139+
columnInfo.setPartition(true);
140+
columnInfo.setType("string");
141+
columnList.add(columnInfo);
142+
}
143+
}
144+
tableInfo.setColumns(columnList);
145+
146+
// set terminated symbol
147+
if (hiveInfo.getDataSeparator() != null) {
148+
char ch = (char) Integer.parseInt(hiveInfo.getDataSeparator());
149+
tableInfo.setFieldTerSymbol(String.valueOf(ch));
150+
}
151+
152+
return tableInfo;
153+
}
154+
122155
}

0 commit comments

Comments
 (0)