From e46b7463d638f33d23d527666e8a13102c94487d Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 2 Jan 2025 22:47:37 +0800 Subject: [PATCH 1/9] [fix](hive) sync trucate table command to other FE --- .../java/org/apache/doris/catalog/Env.java | 9 ++++++- .../doris/datasource/ExternalCatalog.java | 17 ++++++++++-- .../datasource/hive/HiveMetadataOps.java | 26 +++++++++++++----- .../operations/ExternalMetadataOps.java | 2 +- .../doris/persist/TruncateTableInfo.java | 27 +++++++++++++++++-- 5 files changed, 69 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 7e0617a84a0a54..d2bc837884e8e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -5858,7 +5858,14 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { } public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundException { - getInternalCatalog().replayTruncateTable(info); + if (Strings.isNullOrEmpty(info.getCtl())) { + getInternalCatalog().replayTruncateTable(info); + } else { + ExternalCatalog ctl = catalogMgr.getCatalog(info.getCtl()); + if (ctl != null) { + ctl.replayTruncateTable(info); + } + } } public void createFunction(CreateFunctionStmt stmt) throws UserException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 05ad1c07cb0991..1db4de6a3c4565 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -59,6 +59,7 @@ import org.apache.doris.datasource.test.TestExternalDatabase; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase; import org.apache.doris.fs.remote.dfs.DFSFileSystem; +import org.apache.doris.persist.TruncateTableInfo; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; @@ -1074,13 +1075,25 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { if (tableRef.getPartitionNames() != null) { partitions = tableRef.getPartitionNames().getPartitionNames(); } - metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(), partitions); + metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(), partitions, false); } catch (Exception e) { - LOG.warn("Failed to drop a table", e); + LOG.warn("Failed to truncate table {}.{} in catlaog {}", stmt.getTblRef().getName().getDb(), + stmt.getTblRef().getName().getTbl(), getName(), e); throw e; } } + public void replayTruncateTable(TruncateTableInfo info) { + if (metadataOps != null) { + try { + metadataOps.truncateTable(info.getDb(), info.getTable(), info.getExtPartNames(), true); + } catch (DdlException e) { + LOG.warn("Failed to replay truncate table {}.{} in catalog {}", info.getDb(), info.getTable(), + getName(), e); + } + } + } + public void setAutoAnalyzePolicy(String dbName, String tableName, String policy) { Pair key = Pair.of(dbName, tableName); if (policy == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index b44ffe00390536..fb262df78b0342 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -40,6 +40,7 @@ import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.persist.TruncateTableInfo; import org.apache.doris.qe.ConnectContext; import com.google.common.annotations.VisibleForTesting; @@ -306,16 +307,29 @@ public void dropTable(DropTableStmt stmt) throws DdlException { } @Override - public void truncateTable(String dbName, String tblName, List partitions) throws DdlException { + public void truncateTable(String dbName, String tblName, List partitions, boolean isReplay) + throws DdlException { ExternalDatabase db = catalog.getDbNullable(dbName); if (db == null) { - throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName()); + if (!isReplay) { + throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName()); + } else { + LOG.warn("Failed to get database: '{}' in catalog: {} when replaying truncate table", dbName, + catalog.getName()); + return; + } } - try { - client.truncateTable(dbName, tblName, partitions); - } catch (Exception e) { - throw new DdlException(e.getMessage(), e); + if (!isReplay) { + try { + client.truncateTable(dbName, tblName, partitions); + } catch (Exception e) { + throw new DdlException(e.getMessage(), e); + } + TruncateTableInfo info = new TruncateTableInfo(catalog.getName(), dbName, tblName, partitions); + Env.getCurrentEnv().getEditLog().logTruncateTable(info); } + + // Invalidate cache. Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(catalog.getId(), dbName, tblName); db.setLastUpdateTime(System.currentTimeMillis()); db.setUnInitialized(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java index e5ed129c679ffe..87b81f78becd29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java @@ -66,7 +66,7 @@ public interface ExternalMetadataOps { * @param tblName * @param partitions */ - void truncateTable(String dbName, String tblName, List partitions) throws DdlException; + void truncateTable(String dbName, String tblName, List partitions, boolean isReplay) throws DdlException; /** * diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java index ffb59d2afdffd9..0e00139638c5aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java @@ -33,6 +33,8 @@ import java.util.Map; public class TruncateTableInfo implements Writable { + @SerializedName(value = "dbId") + private String ctl; @SerializedName(value = "dbId") private long dbId; @SerializedName(value = "db") @@ -43,6 +45,9 @@ public class TruncateTableInfo implements Writable { private String table; @SerializedName(value = "partitions") private List partitions = Lists.newArrayList(); + // Only for external table + @SerializedName(value = "extParts") + private List extPartNames = Lists.newArrayList(); @SerializedName(value = "isEntireTable") private boolean isEntireTable = false; @SerializedName(value = "rawSql") @@ -56,6 +61,7 @@ public TruncateTableInfo() { } + // for internal table public TruncateTableInfo(long dbId, String db, long tblId, String table, List partitions, boolean isEntireTable, String rawSql, List oldPartitions, boolean force) { this.dbId = dbId; @@ -71,6 +77,18 @@ public TruncateTableInfo(long dbId, String db, long tblId, String table, List partNames) { + this.ctl = ctl; + this.db = db; + this.table = table; + this.extPartNames = partNames; + } + + public String getCtl() { + return ctl; + } + public long getDbId() { return dbId; } @@ -91,6 +109,10 @@ public List getPartitions() { return partitions; } + public List getExtPartNames() { + return extPartNames; + } + public Map getOldPartitions() { return oldPartitions == null ? new HashMap<>() : oldPartitions; } @@ -103,7 +125,6 @@ public boolean getForce() { return force; } - public String getRawSql() { return rawSql; } @@ -126,13 +147,15 @@ public String toJson() { @Override public String toString() { return "TruncateTableInfo{" + + "ctl=" + ctl + "dbId=" + dbId + ", db='" + db + '\'' + ", tblId=" + tblId + ", table='" + table + '\'' + ", isEntireTable=" + isEntireTable + ", rawSql='" + rawSql + '\'' - + ", partitions_size=" + partitions.size() + + ", partitions_size=" + (partitions == null ? "0" : partitions.size()) + + ", extPartNames_size=" + (extPartNames == null ? "0" : extPartNames.size()) + '}'; } } From 69ed3ef7770489ffc2cb6ea394de4ee161164fc7 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 2 Jan 2025 22:51:37 +0800 Subject: [PATCH 2/9] 1 --- fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index d2bc837884e8e6..86b420f7d490f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -5859,9 +5859,11 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundException { if (Strings.isNullOrEmpty(info.getCtl())) { + // In previous versions(before 2.1.8), there is no catalog info in TruncateTableInfo, + // So if the catalog info is empty, we assume it's internal table. getInternalCatalog().replayTruncateTable(info); } else { - ExternalCatalog ctl = catalogMgr.getCatalog(info.getCtl()); + ExternalCatalog ctl = (ExternalCatalog) catalogMgr.getCatalog(info.getCtl()); if (ctl != null) { ctl.replayTruncateTable(info); } From ddad201d86e06902886697662fd9956e3a22db3b Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 2 Jan 2025 22:53:15 +0800 Subject: [PATCH 3/9] 2 --- .../main/java/org/apache/doris/datasource/ExternalCatalog.java | 2 +- .../org/apache/doris/datasource/iceberg/IcebergMetadataOps.java | 2 +- .../main/java/org/apache/doris/persist/TruncateTableInfo.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 1db4de6a3c4565..3948080ff9a49a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1077,7 +1077,7 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { } metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(), partitions, false); } catch (Exception e) { - LOG.warn("Failed to truncate table {}.{} in catlaog {}", stmt.getTblRef().getName().getDb(), + LOG.warn("Failed to truncate table {}.{} in catalog {}", stmt.getTblRef().getName().getDb(), stmt.getTblRef().getName().getTbl(), getName(), e); throw e; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index bf07284a6d80ee..cbd71a2dac4abe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -261,7 +261,7 @@ private void performDropTable(DropTableStmt stmt) throws DdlException { } @Override - public void truncateTable(String dbName, String tblName, List partitions) { + public void truncateTable(String dbName, String tblName, List partitions, boolean isReplay) { throw new UnsupportedOperationException("Truncate Iceberg table is not supported."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java index 0e00139638c5aa..10d1e07074a811 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java @@ -33,7 +33,7 @@ import java.util.Map; public class TruncateTableInfo implements Writable { - @SerializedName(value = "dbId") + @SerializedName(value = "ctl") private String ctl; @SerializedName(value = "dbId") private long dbId; From 512d11dc6f014217a67882060ffd060ac8406990 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 3 Jan 2025 00:03:14 +0800 Subject: [PATCH 4/9] 1 --- .../java/org/apache/doris/catalog/Env.java | 36 ++++++++- .../doris/datasource/ExternalCatalog.java | 50 +++++++++++-- .../datasource/hive/HiveMetadataOps.java | 72 +++++++++++------- .../iceberg/IcebergMetadataOps.java | 39 +++++++--- .../operations/ExternalMetadataOps.java | 52 +++++++++++-- .../apache/doris/journal/JournalEntity.java | 6 ++ .../apache/doris/persist/CreateDbInfo.java | 73 +++++++++++++++++++ .../apache/doris/persist/CreateTableInfo.java | 21 ++++++ .../org/apache/doris/persist/DropDbInfo.java | 13 ++++ .../org/apache/doris/persist/DropInfo.java | 19 +++++ .../org/apache/doris/persist/EditLog.java | 57 +++++++++++---- .../apache/doris/persist/OperationType.java | 1 + 12 files changed, 375 insertions(+), 64 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 86b420f7d490f5..fd093af58cbafd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -211,6 +211,9 @@ import org.apache.doris.persist.BackendTabletsInfo; import org.apache.doris.persist.BinlogGcInfo; import org.apache.doris.persist.CleanQueryStatsInfo; +import org.apache.doris.persist.CreateDbInfo; +import org.apache.doris.persist.CreateTableInfo; +import org.apache.doris.persist.DropDbInfo; import org.apache.doris.persist.DropPartitionInfo; import org.apache.doris.persist.EditLog; import org.apache.doris.persist.GlobalVarPersistInfo; @@ -3263,6 +3266,17 @@ public void replayCreateDb(Database db) { getInternalCatalog().replayCreateDb(db, ""); } + public void replayNewCreateDb(CreateDbInfo info) { + if (info.getCtlName().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { + getInternalCatalog().replayCreateDb(info.getInternalDb(), ""); + } else { + ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); + if (externalCatalog != null) { + externalCatalog.replayCreateDb(info.getDbName()); + } + } + } + public void dropDb(DropDbStmt stmt) throws DdlException { CatalogIf catalogIf; if (StringUtils.isEmpty(stmt.getCtlName())) { @@ -3273,8 +3287,15 @@ public void dropDb(DropDbStmt stmt) throws DdlException { catalogIf.dropDb(stmt); } - public void replayDropDb(String dbName, boolean isForceDrop, Long recycleTime) throws DdlException { - getInternalCatalog().replayDropDb(dbName, isForceDrop, recycleTime); + public void replayDropDb(DropDbInfo info) throws DdlException { + if (Strings.isNullOrEmpty(info.getCtlName())) { + getInternalCatalog().replayDropDb(info.getDbName(), info.isForceDrop(), info.getRecycleTime()); + } else { + ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); + if (externalCatalog != null) { + externalCatalog.replayDropDb(info.getDbName()); + } + } } public void recoverDatabase(RecoverDbStmt recoverStmt) throws DdlException { @@ -4153,8 +4174,15 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis } } - public void replayCreateTable(String dbName, Table table) throws MetaNotFoundException { - getInternalCatalog().replayCreateTable(dbName, table); + public void replayCreateTable(CreateTableInfo info) throws MetaNotFoundException { + if (Strings.isNullOrEmpty(info.getCtlName())) { + getInternalCatalog().replayCreateTable(info.getDbName(), info.getTable()); + } else { + ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); + if (externalCatalog != null) { + externalCatalog.replayCreateTable(info.getDbName(), info.getTblName()); + } + } } public void replayAlterExternalTableSchema(String dbName, String tableName, List newSchema) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 3948080ff9a49a..6c82804d864c22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -59,6 +59,10 @@ import org.apache.doris.datasource.test.TestExternalDatabase; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase; import org.apache.doris.fs.remote.dfs.DFSFileSystem; +import org.apache.doris.persist.CreateDbInfo; +import org.apache.doris.persist.CreateTableInfo; +import org.apache.doris.persist.DropDbInfo; +import org.apache.doris.persist.DropInfo; import org.apache.doris.persist.TruncateTableInfo; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -927,12 +931,20 @@ public void createDb(CreateDbStmt stmt) throws DdlException { } try { metadataOps.createDb(stmt); + CreateDbInfo info = new CreateDbInfo(getName(), stmt.getFullDbName(), null); + Env.getCurrentEnv().getEditLog().logNewCreateDb(info); } catch (Exception e) { - LOG.warn("Failed to create a database.", e); + LOG.warn("Failed to create database {} in catalog {}.", stmt.getFullDbName(), getName(), e); throw e; } } + public void replayCreateDb(String dbName) { + if (metadataOps != null) { + metadataOps.afterCreateDb(dbName); + } + } + @Override public void dropDb(DropDbStmt stmt) throws DdlException { makeSureInitialized(); @@ -942,12 +954,20 @@ public void dropDb(DropDbStmt stmt) throws DdlException { } try { metadataOps.dropDb(stmt); + DropDbInfo info = new DropDbInfo(getName(), stmt.getDbName()); + Env.getCurrentEnv().getEditLog().logDropDb(info); } catch (Exception e) { - LOG.warn("Failed to drop a database.", e); + LOG.warn("Failed to drop database {} in catalog {}", stmt.getDbName(), getName(), e); throw e; } } + public void replayDropDb(String dbName) { + if (metadataOps != null) { + metadataOps.afterDropDb(dbName); + } + } + @Override public boolean createTable(CreateTableStmt stmt) throws UserException { makeSureInitialized(); @@ -956,13 +976,25 @@ public boolean createTable(CreateTableStmt stmt) throws UserException { return false; } try { - return metadataOps.createTable(stmt); + boolean res = metadataOps.createTable(stmt); + if (!res) { + // res == false means the table does not exist before, and we create it. + CreateTableInfo info = new CreateTableInfo(getName(), stmt.getDbName(), stmt.getTableName()); + Env.getCurrentEnv().getEditLog().logCreateTable(info); + } + return res; } catch (Exception e) { LOG.warn("Failed to create a table.", e); throw e; } } + public void replayCreateTable(String dbName, String tblName) { + if (metadataOps != null) { + metadataOps.afterCreateTable(dbName, tblName); + } + } + @Override public void dropTable(DropTableStmt stmt) throws DdlException { makeSureInitialized(); @@ -972,12 +1004,20 @@ public void dropTable(DropTableStmt stmt) throws DdlException { } try { metadataOps.dropTable(stmt); + DropInfo info = new DropInfo(getName(), stmt.getDbName(), stmt.getTableName()); + Env.getCurrentEnv().getEditLog().logDropTable(info); } catch (Exception e) { LOG.warn("Failed to drop a table", e); throw e; } } + public void replayDropTable(String dbName, String tblName) { + if (metadataOps != null) { + metadataOps.afterDropTable(dbName, tblName); + } + } + public void unregisterDatabase(String dbName) { throw new NotImplementedException("unregisterDatabase not implemented"); } @@ -1075,7 +1115,7 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { if (tableRef.getPartitionNames() != null) { partitions = tableRef.getPartitionNames().getPartitionNames(); } - metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(), partitions, false); + metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(), partitions); } catch (Exception e) { LOG.warn("Failed to truncate table {}.{} in catalog {}", stmt.getTblRef().getName().getDb(), stmt.getTblRef().getName().getTbl(), getName(), e); @@ -1086,7 +1126,7 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { public void replayTruncateTable(TruncateTableInfo info) { if (metadataOps != null) { try { - metadataOps.truncateTable(info.getDb(), info.getTable(), info.getExtPartNames(), true); + metadataOps.truncateTable(info.getDb(), info.getTable(), info.getExtPartNames()); } catch (DdlException e) { LOG.warn("Failed to replay truncate table {}.{} in catalog {}", info.getDb(), info.getTable(), getName(), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index fb262df78b0342..1d8bed011b40ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -108,7 +108,7 @@ private static HMSCachedClient createCachedClient(HiveConf hiveConf, int thriftC } @Override - public void createDb(CreateDbStmt stmt) throws DdlException { + public void createDbImpl(CreateDbStmt stmt) throws DdlException { String fullDbName = stmt.getFullDbName(); Map properties = stmt.getProperties(); long dbId = Env.getCurrentEnv().getNextId(); @@ -131,7 +131,6 @@ public void createDb(CreateDbStmt stmt) throws DdlException { catalogDatabase.setProperties(properties); catalogDatabase.setComment(properties.getOrDefault("comment", "")); client.createDatabase(catalogDatabase); - catalog.onRefreshCache(true); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } @@ -139,7 +138,12 @@ public void createDb(CreateDbStmt stmt) throws DdlException { } @Override - public void dropDb(DropDbStmt stmt) throws DdlException { + public void afterCreateDb(String dbName) { + catalog.onRefreshCache(true); + } + + @Override + public void dropDbImpl(DropDbStmt stmt) throws DdlException { String dbName = stmt.getDbName(); if (!databaseExist(dbName)) { if (stmt.isSetIfExists()) { @@ -151,14 +155,18 @@ public void dropDb(DropDbStmt stmt) throws DdlException { } try { client.dropDatabase(dbName); - catalog.onRefreshCache(true); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } @Override - public boolean createTable(CreateTableStmt stmt) throws UserException { + public void afterDropDb(String dbName) { + catalog.onRefreshCache(true); + } + + @Override + public boolean createTableImpl(CreateTableStmt stmt) throws UserException { String dbName = stmt.getDbName(); String tblName = stmt.getTableName(); ExternalDatabase db = catalog.getDbNullable(dbName); @@ -266,7 +274,6 @@ In hive, this table only can insert not update(not report error,but not actually comment); } client.createTable(hiveTableMeta, stmt.isSetIfNotExists()); - db.setUnInitialized(true); } catch (Exception e) { throw new UserException(e.getMessage(), e); } @@ -274,7 +281,15 @@ In hive, this table only can insert not update(not report error,but not actually } @Override - public void dropTable(DropTableStmt stmt) throws DdlException { + public void afterCreateTable(String dbName, String tblName) { + ExternalDatabase db = catalog.getDbNullable(dbName); + if (db != null) { + db.setUnInitialized(true); + } + } + + @Override + public void dropTableImpl(DropTableStmt stmt) throws DdlException { String dbName = stmt.getDbName(); String tblName = stmt.getTableName(); ExternalDatabase db = catalog.getDbNullable(stmt.getDbName()); @@ -300,39 +315,44 @@ public void dropTable(DropTableStmt stmt) throws DdlException { try { client.dropTable(dbName, tblName); - db.setUnInitialized(true); } catch (Exception e) { throw new DdlException(e.getMessage(), e); } } @Override - public void truncateTable(String dbName, String tblName, List partitions, boolean isReplay) + public void afterDropTable(String dbName, String tblName) { + ExternalDatabase db = catalog.getDbNullable(dbName); + if (db != null) { + db.setUnInitialized(true); + } + } + + @Override + public void truncateTableImpl(String dbName, String tblName, List partitions) throws DdlException { ExternalDatabase db = catalog.getDbNullable(dbName); if (db == null) { - if (!isReplay) { - throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName()); - } else { - LOG.warn("Failed to get database: '{}' in catalog: {} when replaying truncate table", dbName, - catalog.getName()); - return; - } + throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName()); } - if (!isReplay) { - try { - client.truncateTable(dbName, tblName, partitions); - } catch (Exception e) { - throw new DdlException(e.getMessage(), e); - } - TruncateTableInfo info = new TruncateTableInfo(catalog.getName(), dbName, tblName, partitions); - Env.getCurrentEnv().getEditLog().logTruncateTable(info); + try { + client.truncateTable(dbName, tblName, partitions); + } catch (Exception e) { + throw new DdlException(e.getMessage(), e); } + TruncateTableInfo info = new TruncateTableInfo(catalog.getName(), dbName, tblName, partitions); + Env.getCurrentEnv().getEditLog().logTruncateTable(info); + } + @Override + public void afterTruncateTable(String dbName, String tblName) { // Invalidate cache. Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(catalog.getId(), dbName, tblName); - db.setLastUpdateTime(System.currentTimeMillis()); - db.setUnInitialized(true); + ExternalDatabase db = catalog.getDbNullable(dbName); + if (db != null) { + db.setLastUpdateTime(System.currentTimeMillis()); + db.setUnInitialized(true); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index cbd71a2dac4abe..2651b1e2806e80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -117,12 +117,11 @@ public List listTableNames(String dbName) { } @Override - public void createDb(CreateDbStmt stmt) throws DdlException { + public void createDbImpl(CreateDbStmt stmt) throws DdlException { try { preExecutionAuthenticator.execute(() -> { performCreateDb(stmt); return null; - }); } catch (Exception e) { throw new DdlException("Failed to create database: " @@ -130,6 +129,11 @@ public void createDb(CreateDbStmt stmt) throws DdlException { } } + @Override + public void afterCreateDb(String dbName) { + dorisCatalog.onRefreshCache(true); + } + private void performCreateDb(CreateDbStmt stmt) throws DdlException { SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog; String dbName = stmt.getFullDbName(); @@ -150,11 +154,10 @@ private void performCreateDb(CreateDbStmt stmt) throws DdlException { } } nsCatalog.createNamespace(getNamespace(dbName), properties); - dorisCatalog.onRefreshCache(true); } @Override - public void dropDb(DropDbStmt stmt) throws DdlException { + public void dropDbImpl(DropDbStmt stmt) throws DdlException { try { preExecutionAuthenticator.execute(() -> { preformDropDb(stmt); @@ -177,11 +180,15 @@ private void preformDropDb(DropDbStmt stmt) throws DdlException { } } nsCatalog.dropNamespace(getNamespace(dbName)); + } + + @Override + public void afterDropDb(String dbName) { dorisCatalog.onRefreshCache(true); } @Override - public boolean createTable(CreateTableStmt stmt) throws UserException { + public boolean createTableImpl(CreateTableStmt stmt) throws UserException { try { preExecutionAuthenticator.execute(() -> performCreateTable(stmt)); } catch (Exception e) { @@ -218,12 +225,19 @@ public boolean performCreateTable(CreateTableStmt stmt) throws UserException { properties.put(ExternalCatalog.DORIS_VERSION, ExternalCatalog.DORIS_VERSION_VALUE); PartitionSpec partitionSpec = IcebergUtils.solveIcebergPartitionSpec(stmt.getPartitionDesc(), schema); catalog.createTable(getTableIdentifier(dbName, tableName), schema, partitionSpec, properties); - db.setUnInitialized(true); return false; } @Override - public void dropTable(DropTableStmt stmt) throws DdlException { + public void afterCreateTable(String dbName, String tblName) { + ExternalDatabase db = dorisCatalog.getDbNullable(dbName); + if (db != null) { + db.setUnInitialized(true); + } + } + + @Override + public void dropTableImpl(DropTableStmt stmt) throws DdlException { try { preExecutionAuthenticator.execute(() -> { performDropTable(stmt); @@ -235,6 +249,14 @@ public void dropTable(DropTableStmt stmt) throws DdlException { } } + @Override + public void afterDropTable(String dbName, String tblName) { + ExternalDatabase db = dorisCatalog.getDbNullable(dbName); + if (db != null) { + db.setUnInitialized(true); + } + } + private void performDropTable(DropTableStmt stmt) throws DdlException { String dbName = stmt.getDbName(); String tableName = stmt.getTableName(); @@ -257,11 +279,10 @@ private void performDropTable(DropTableStmt stmt) throws DdlException { } } catalog.dropTable(getTableIdentifier(dbName, tableName), true); - db.setUnInitialized(true); } @Override - public void truncateTable(String dbName, String tblName, List partitions, boolean isReplay) { + public void truncateTableImpl(String dbName, String tblName, List partitions) { throw new UnsupportedOperationException("Truncate Iceberg table is not supported."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java index 87b81f78becd29..f565f2795e8448 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java @@ -36,14 +36,29 @@ public interface ExternalMetadataOps { * @param stmt * @throws DdlException */ - void createDb(CreateDbStmt stmt) throws DdlException; + default void createDb(CreateDbStmt stmt) throws DdlException { + createDbImpl(stmt); + afterCreateDb(stmt.getFullDbName()); + } + + void createDbImpl(CreateDbStmt stmt) throws DdlException; + + default void afterCreateDb(String dbName) { + } /** * drop db in external metastore * @param stmt * @throws DdlException */ - void dropDb(DropDbStmt stmt) throws DdlException; + default void dropDb(DropDbStmt stmt) throws DdlException { + dropDbImpl(stmt); + afterDropDb(stmt.getCtlName()); + } + + void dropDbImpl(DropDbStmt stmt) throws DdlException; + + void afterDropDb(String dbName); /** * @@ -51,14 +66,33 @@ public interface ExternalMetadataOps { * @return if set isExists is true, return true if table exists, otherwise return false * @throws UserException */ - boolean createTable(CreateTableStmt stmt) throws UserException; + default boolean createTable(CreateTableStmt stmt) throws UserException { + boolean res = createTableImpl(stmt); + if (!res) { + afterCreateTable(stmt.getDbName(), stmt.getTableName()); + } + return res; + } + + boolean createTableImpl(CreateTableStmt stmt) throws UserException; + + default void afterCreateTable(String dbName, String tblName) { + } /** * * @param stmt * @throws DdlException */ - void dropTable(DropTableStmt stmt) throws DdlException; + default void dropTable(DropTableStmt stmt) throws DdlException { + dropTableImpl(stmt); + afterDropTable(stmt.getDbName(), stmt.getTableName()); + } + + void dropTableImpl(DropTableStmt stmt) throws DdlException; + + default void afterDropTable(String dbName, String tblName) { + } /** * @@ -66,7 +100,15 @@ public interface ExternalMetadataOps { * @param tblName * @param partitions */ - void truncateTable(String dbName, String tblName, List partitions, boolean isReplay) throws DdlException; + default void truncateTable(String dbName, String tblName, List partitions) throws DdlException { + truncateTableImpl(dbName, tblName, partitions); + afterTruncateTable(dbName, tblName); + } + + void truncateTableImpl(String dbName, String tblName, List partitions) throws DdlException; + + default void afterTruncateTable(String dbName, String tblName) { + } /** * diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 7527755baab0f1..29096b68ce00bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -81,6 +81,7 @@ import org.apache.doris.persist.CleanQueryStatsInfo; import org.apache.doris.persist.ColocatePersistInfo; import org.apache.doris.persist.ConsistencyCheckInfo; +import org.apache.doris.persist.CreateDbInfo; import org.apache.doris.persist.CreateTableInfo; import org.apache.doris.persist.DatabaseInfo; import org.apache.doris.persist.DropDbInfo; @@ -217,6 +218,11 @@ public void readFields(DataInput in) throws IOException { isRead = true; break; } + case OperationType.OP_NEW_CREATE_DB: { + data = CreateDbInfo.read(in); + isRead = true; + break; + } case OperationType.OP_DROP_DB: { data = DropDbInfo.read(in); isRead = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java new file mode 100644 index 00000000000000..522ae02142915a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist; + +import org.apache.doris.catalog.Database; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class CreateDbInfo implements Writable { + + @SerializedName("ctl") + private String ctlName; + @SerializedName("db") + private String dbName; + @SerializedName("idb") + private Database internalDb; + + public CreateDbInfo() { + this.ctlName = ""; + this.dbName = ""; + } + + public CreateDbInfo(String ctlName, String dbName, Database internalDb) { + this.ctlName = ctlName; + this.dbName = dbName; + this.internalDb = internalDb; + } + + public String getCtlName() { + return ctlName; + } + + public String getDbName() { + return dbName; + } + + public Database getInternalDb() { + return internalDb; + } + + public static CreateDbInfo read(DataInput in) throws IOException { + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, CreateDbInfo.class); + } + + @Override + public void write(DataOutput out) throws IOException { + String json = GsonUtils.GSON.toJson(this); + Text.writeString(out, json); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java index 1f2bcc15eb4252..87ad4d7ad9974c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java @@ -38,8 +38,12 @@ public class CreateTableInfo implements Writable, GsonPostProcessable { public static final Logger LOG = LoggerFactory.getLogger(CreateTableInfo.class); + @SerializedName(value = "ctl") + private String ctlName; @SerializedName(value = "dbName") private String dbName; + @SerializedName(value = "tbl") + private String tblName; @SerializedName(value = "table") private Table table; @@ -47,15 +51,32 @@ public CreateTableInfo() { // for persist } + // for internal table public CreateTableInfo(String dbName, Table table) { this.dbName = dbName; + this.tblName = table.getName(); this.table = table; } + // for external table + public CreateTableInfo(String ctlName, String dbName, String tblName) { + this.ctlName = ctlName; + this.dbName = dbName; + this.tblName = tblName; + } + + public String getCtlName() { + return ctlName; + } + public String getDbName() { return dbName; } + public String getTblName() { + return tblName; + } + public Table getTable() { return table; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java index 190db6a671aa0d..f9d7339ca41cd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java @@ -30,6 +30,8 @@ import java.io.IOException; public class DropDbInfo implements Writable, GsonPostProcessable { + @SerializedName(value = "ctl") + private String ctlName; @SerializedName(value = "dbName") private String dbName; @SerializedName(value = "forceDrop") @@ -41,12 +43,23 @@ public DropDbInfo() { this("", false, 0); } + // for external table + public DropDbInfo(String ctlName, String dbName) { + this.ctlName = ctlName; + this.dbName = dbName; + } + + // for internal table public DropDbInfo(String dbName, boolean forceDrop, long recycleTime) { this.dbName = dbName; this.forceDrop = forceDrop; this.recycleTime = recycleTime; } + public String getCtlName() { + return ctlName; + } + public String getDbName() { return dbName; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java index db0688bd6ad267..a1e679a18dc99b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java @@ -30,6 +30,10 @@ import java.io.IOException; public class DropInfo implements Writable { + @SerializedName(value = "ctl") + private String ctl; + @SerializedName(value = "db") + private String db; @SerializedName(value = "dbId") private long dbId; @SerializedName(value = "tableId") @@ -50,6 +54,13 @@ public class DropInfo implements Writable { public DropInfo() { } + // for external table + public DropInfo(String ctl, String db, String tbl) { + this.ctl = ctl; + this.db = db; + this.tableName = tbl; + } + public DropInfo(long dbId, long tableId, String tableName, boolean isView, boolean forceDrop, long recycleTime) { this(dbId, tableId, tableName, -1L, "", isView, forceDrop, recycleTime); @@ -67,6 +78,14 @@ public DropInfo(long dbId, long tableId, String tableName, long indexId, String this.recycleTime = recycleTime; } + public String getCtl() { + return ctl; + } + + public String getDb() { + return db; + } + public long getDbId() { return this.dbId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 65a161157c8967..ee68dd3c18e25b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -51,6 +51,7 @@ import org.apache.doris.cooldown.CooldownConfList; import org.apache.doris.cooldown.CooldownDelete; import org.apache.doris.datasource.CatalogLog; +import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalObjectLog; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.InitDatabaseLog; @@ -100,6 +101,7 @@ import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; +import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -194,9 +196,14 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { env.replayCreateDb(db); break; } + case OperationType.OP_NEW_CREATE_DB: { + CreateDbInfo info = (CreateDbInfo) journal.getData(); + env.replayCreateDb(info); + break; + } case OperationType.OP_DROP_DB: { DropDbInfo dropDbInfo = (DropDbInfo) journal.getData(); - env.replayDropDb(dropDbInfo.getDbName(), dropDbInfo.isForceDrop(), dropDbInfo.getRecycleTime()); + env.replayDropDb(dropDbInfo); break; } case OperationType.OP_ALTER_DB: { @@ -227,9 +234,11 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { CreateTableInfo info = (CreateTableInfo) journal.getData(); LOG.info("Begin to unprotect create table. db = " + info.getDbName() + " table = " + info.getTable() .getId()); - CreateTableRecord record = new CreateTableRecord(logId, info); - env.replayCreateTable(info.getDbName(), info.getTable()); - env.getBinlogManager().addCreateTableRecord(record); + env.replayCreateTable(info); + if (Strings.isNullOrEmpty(info.getCtlName())) { + CreateTableRecord record = new CreateTableRecord(logId, info); + env.getBinlogManager().addCreateTableRecord(record); + } break; } case OperationType.OP_ALTER_EXTERNAL_TABLE_SCHEMA: { @@ -241,12 +250,20 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { } case OperationType.OP_DROP_TABLE: { DropInfo info = (DropInfo) journal.getData(); - Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(info.getDbId()); - LOG.info("Begin to unprotect drop table. db = " + db.getFullName() + " table = " - + info.getTableId()); - DropTableRecord record = new DropTableRecord(logId, info); - env.replayDropTable(db, info.getTableId(), info.isForceDrop(), info.getRecycleTime()); - env.getBinlogManager().addDropTableRecord(record); + if (Strings.isNullOrEmpty(info.getCtl())) { + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(info.getDbId()); + LOG.info("Begin to unprotect drop table. db = " + db.getFullName() + " table = " + + info.getTableId()); + env.replayDropTable(db, info.getTableId(), info.isForceDrop(), info.getRecycleTime()); + DropTableRecord record = new DropTableRecord(logId, info); + env.getBinlogManager().addDropTableRecord(record); + } else { + ExternalCatalog ctl = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr() + .getCatalog(info.getCtl()); + if (ctl != null) { + ctl.replayDropTable(info.getDb(), info.getTableName()); + } + } break; } case OperationType.OP_ADD_PARTITION: { @@ -1402,6 +1419,10 @@ public void logCreateDb(Database db) { logEdit(OperationType.OP_CREATE_DB, db); } + public void logNewCreateDb(CreateDbInfo info) { + logEdit(OperationType.OP_NEW_CREATE_DB, info); + } + public void logDropDb(DropDbInfo dropDbInfo) { logEdit(OperationType.OP_DROP_DB, dropDbInfo); } @@ -1420,8 +1441,10 @@ public void logAlterDb(DatabaseInfo dbInfo) { public void logCreateTable(CreateTableInfo info) { long logId = logEdit(OperationType.OP_CREATE_TABLE, info); - CreateTableRecord record = new CreateTableRecord(logId, info); - Env.getCurrentEnv().getBinlogManager().addCreateTableRecord(record); + if (Strings.isNullOrEmpty(info.getCtlName())) { + CreateTableRecord record = new CreateTableRecord(logId, info); + Env.getCurrentEnv().getBinlogManager().addCreateTableRecord(record); + } } public void logRefreshExternalTableSchema(RefreshExternalTableInfo info) { @@ -1465,8 +1488,10 @@ public void logBatchModifyPartition(BatchModifyPartitionsInfo info) { public void logDropTable(DropInfo info) { long logId = logEdit(OperationType.OP_DROP_TABLE, info); - DropTableRecord record = new DropTableRecord(logId, info); - Env.getCurrentEnv().getBinlogManager().addDropTableRecord(record); + if (Strings.isNullOrEmpty(info.getCtl())) { + DropTableRecord record = new DropTableRecord(logId, info); + Env.getCurrentEnv().getBinlogManager().addDropTableRecord(record); + } } public void logEraseTable(long tableId) { @@ -1704,7 +1729,9 @@ public void logUpdateUserProperty(UserPropertyInfo propertyInfo) { public void logTruncateTable(TruncateTableInfo info) { long logId = logEdit(OperationType.OP_TRUNCATE_TABLE, info); LOG.info("log truncate table, logId:{}, infos: {}", logId, info); - Env.getCurrentEnv().getBinlogManager().addTruncateTable(info, logId); + if (Strings.isNullOrEmpty(info.getCtl())) { + Env.getCurrentEnv().getBinlogManager().addTruncateTable(info, logId); + } } public void logColocateModifyRepliaAlloc(ColocatePersistInfo info) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 6c97606cb24ab8..0c21533d6b25ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -33,6 +33,7 @@ public class OperationType { public static final short OP_ERASE_DB = 4; public static final short OP_RECOVER_DB = 5; public static final short OP_RENAME_DB = 6; + public static final short OP_NEW_CREATE_DB = 7; // 10~19 110~119 210~219 ... public static final short OP_CREATE_TABLE = 10; From a073b85b8900d1418e087e7ad9e3dd3e0a0de27a Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 9 Jan 2025 21:55:05 +0800 Subject: [PATCH 5/9] 4 --- .../src/main/java/org/apache/doris/catalog/Env.java | 11 +++++++++-- .../main/java/org/apache/doris/persist/EditLog.java | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index fd093af58cbafd..0de2f53e3b57f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3262,8 +3262,15 @@ public void unprotectCreateDb(Database db) { getInternalCatalog().unprotectCreateDb(db); } - public void replayCreateDb(Database db) { - getInternalCatalog().replayCreateDb(db, ""); + public void replayCreateDb(CreateDbInfo dbInfo) { + if (dbInfo.getInternalDb() != null) { + getInternalCatalog().replayCreateDb(dbInfo.getInternalDb(), ""); + } else { + ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(dbInfo.getCtlName()); + if (externalCatalog != null) { + externalCatalog.replayCreateDb(dbInfo.getDbName()); + } + } } public void replayNewCreateDb(CreateDbInfo info) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index ee68dd3c18e25b..f26050ac6544ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -193,7 +193,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { } case OperationType.OP_CREATE_DB: { Database db = (Database) journal.getData(); - env.replayCreateDb(db); + CreateDbInfo info = new CreateDbInfo(db.getCatalog().getName(), db.getName(), db); + env.replayCreateDb(info); break; } case OperationType.OP_NEW_CREATE_DB: { From faab2b62bdc78e6361c114bac5b90cb357ab4276 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 9 Jan 2025 22:45:32 +0800 Subject: [PATCH 6/9] 5 --- .../main/java/org/apache/doris/catalog/Env.java | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 0de2f53e3b57f2..1d543f62ec6e85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3273,17 +3273,6 @@ public void replayCreateDb(CreateDbInfo dbInfo) { } } - public void replayNewCreateDb(CreateDbInfo info) { - if (info.getCtlName().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { - getInternalCatalog().replayCreateDb(info.getInternalDb(), ""); - } else { - ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); - if (externalCatalog != null) { - externalCatalog.replayCreateDb(info.getDbName()); - } - } - } - public void dropDb(DropDbStmt stmt) throws DdlException { CatalogIf catalogIf; if (StringUtils.isEmpty(stmt.getCtlName())) { @@ -3295,7 +3284,8 @@ public void dropDb(DropDbStmt stmt) throws DdlException { } public void replayDropDb(DropDbInfo info) throws DdlException { - if (Strings.isNullOrEmpty(info.getCtlName())) { + if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName() + .equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { getInternalCatalog().replayDropDb(info.getDbName(), info.isForceDrop(), info.getRecycleTime()); } else { ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); @@ -5893,7 +5883,7 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { } public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundException { - if (Strings.isNullOrEmpty(info.getCtl())) { + if (Strings.isNullOrEmpty(info.getCtl()) || info.getCtl().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { // In previous versions(before 2.1.8), there is no catalog info in TruncateTableInfo, // So if the catalog info is empty, we assume it's internal table. getInternalCatalog().replayTruncateTable(info); From fd1a613d885a43a7ab1cbf8ff04d9efc2dddc266 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 9 Jan 2025 23:05:08 +0800 Subject: [PATCH 7/9] 6 --- .../java/org/apache/doris/catalog/Env.java | 3 ++- .../doris/datasource/ExternalCatalog.java | 10 ++++----- .../datasource/hive/HiveMetadataOps.java | 3 --- .../org/apache/doris/persist/EditLog.java | 22 +++++++++++-------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 1d543f62ec6e85..b2ec80839387b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -4172,7 +4172,8 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis } public void replayCreateTable(CreateTableInfo info) throws MetaNotFoundException { - if (Strings.isNullOrEmpty(info.getCtlName())) { + if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName() + .equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { getInternalCatalog().replayCreateTable(info.getDbName(), info.getTable()); } else { ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 6c82804d864c22..cd98284638c684 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1116,6 +1116,9 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { partitions = tableRef.getPartitionNames().getPartitionNames(); } metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(), partitions); + TruncateTableInfo info = new TruncateTableInfo(getName(), tableName.getDb(), tableName.getTbl(), + partitions); + Env.getCurrentEnv().getEditLog().logTruncateTable(info); } catch (Exception e) { LOG.warn("Failed to truncate table {}.{} in catalog {}", stmt.getTblRef().getName().getDb(), stmt.getTblRef().getName().getTbl(), getName(), e); @@ -1125,12 +1128,7 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { public void replayTruncateTable(TruncateTableInfo info) { if (metadataOps != null) { - try { - metadataOps.truncateTable(info.getDb(), info.getTable(), info.getExtPartNames()); - } catch (DdlException e) { - LOG.warn("Failed to replay truncate table {}.{} in catalog {}", info.getDb(), info.getTable(), - getName(), e); - } + metadataOps.afterTruncateTable(info.getDb(), info.getTable()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index 1d8bed011b40ff..a52e2d7b57c3dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -40,7 +40,6 @@ import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.property.constants.HMSProperties; -import org.apache.doris.persist.TruncateTableInfo; import org.apache.doris.qe.ConnectContext; import com.google.common.annotations.VisibleForTesting; @@ -340,8 +339,6 @@ public void truncateTableImpl(String dbName, String tblName, List partit } catch (Exception e) { throw new DdlException(e.getMessage(), e); } - TruncateTableInfo info = new TruncateTableInfo(catalog.getName(), dbName, tblName, partitions); - Env.getCurrentEnv().getEditLog().logTruncateTable(info); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index f26050ac6544ba..11a5a58e42bf01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -55,6 +55,7 @@ import org.apache.doris.datasource.ExternalObjectLog; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.InitDatabaseLog; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.MetaIdMappingsLog; import org.apache.doris.ha.MasterInfo; import org.apache.doris.insertoverwrite.InsertOverwriteLog; @@ -236,7 +237,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { LOG.info("Begin to unprotect create table. db = " + info.getDbName() + " table = " + info.getTable() .getId()); env.replayCreateTable(info); - if (Strings.isNullOrEmpty(info.getCtlName())) { + if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName().equals( + InternalCatalog.INTERNAL_CATALOG_NAME)) { CreateTableRecord record = new CreateTableRecord(logId, info); env.getBinlogManager().addCreateTableRecord(record); } @@ -244,17 +246,18 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { } case OperationType.OP_ALTER_EXTERNAL_TABLE_SCHEMA: { RefreshExternalTableInfo info = (RefreshExternalTableInfo) journal.getData(); - LOG.info("Begin to unprotect alter external table schema. db = " + info.getDbName() + " table = " - + info.getTableName()); + LOG.info("Begin to unprotect alter external table schema. db = {} table = {}", info.getDbName(), + info.getTableName()); env.replayAlterExternalTableSchema(info.getDbName(), info.getTableName(), info.getNewSchema()); break; } case OperationType.OP_DROP_TABLE: { DropInfo info = (DropInfo) journal.getData(); - if (Strings.isNullOrEmpty(info.getCtl())) { + if (Strings.isNullOrEmpty(info.getCtl()) || info.getCtl().equals( + InternalCatalog.INTERNAL_CATALOG_NAME)) { Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(info.getDbId()); - LOG.info("Begin to unprotect drop table. db = " + db.getFullName() + " table = " - + info.getTableId()); + LOG.info("Begin to unprotect drop table. db = {} table = {}", db.getFullName(), + info.getTableId()); env.replayDropTable(db, info.getTableId(), info.isForceDrop(), info.getRecycleTime()); DropTableRecord record = new DropTableRecord(logId, info); env.getBinlogManager().addDropTableRecord(record); @@ -1442,7 +1445,8 @@ public void logAlterDb(DatabaseInfo dbInfo) { public void logCreateTable(CreateTableInfo info) { long logId = logEdit(OperationType.OP_CREATE_TABLE, info); - if (Strings.isNullOrEmpty(info.getCtlName())) { + if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName() + .equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { CreateTableRecord record = new CreateTableRecord(logId, info); Env.getCurrentEnv().getBinlogManager().addCreateTableRecord(record); } @@ -1489,7 +1493,7 @@ public void logBatchModifyPartition(BatchModifyPartitionsInfo info) { public void logDropTable(DropInfo info) { long logId = logEdit(OperationType.OP_DROP_TABLE, info); - if (Strings.isNullOrEmpty(info.getCtl())) { + if (Strings.isNullOrEmpty(info.getCtl()) || info.getCtl().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { DropTableRecord record = new DropTableRecord(logId, info); Env.getCurrentEnv().getBinlogManager().addDropTableRecord(record); } @@ -1730,7 +1734,7 @@ public void logUpdateUserProperty(UserPropertyInfo propertyInfo) { public void logTruncateTable(TruncateTableInfo info) { long logId = logEdit(OperationType.OP_TRUNCATE_TABLE, info); LOG.info("log truncate table, logId:{}, infos: {}", logId, info); - if (Strings.isNullOrEmpty(info.getCtl())) { + if (Strings.isNullOrEmpty(info.getCtl()) || info.getCtl().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { Env.getCurrentEnv().getBinlogManager().addTruncateTable(info, logId); } } From 2710d6d0f627a01ff47bb0a67957f25b60ab14ab Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 9 Jan 2025 23:19:39 +0800 Subject: [PATCH 8/9] 7 --- .../doris/datasource/ExternalCatalog.java | 2 +- .../doris/datasource/InternalCatalog.java | 4 +- .../org/apache/doris/persist/EditLog.java | 6 +- .../apache/doris/persist/OperationType.java | 3 +- .../doris/persist/CreateDbInfoTest.java | 72 +++++++++++++++++++ 5 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/persist/CreateDbInfoTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index cd98284638c684..7c755964f8f9dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -932,7 +932,7 @@ public void createDb(CreateDbStmt stmt) throws DdlException { try { metadataOps.createDb(stmt); CreateDbInfo info = new CreateDbInfo(getName(), stmt.getFullDbName(), null); - Env.getCurrentEnv().getEditLog().logNewCreateDb(info); + Env.getCurrentEnv().getEditLog().logCreateDb(info); } catch (Exception e) { LOG.warn("Failed to create database {} in catalog {}.", stmt.getFullDbName(), getName(), e); throw e; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 5b3c106bbc8e52..b0d71e8ebc7189 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -152,6 +152,7 @@ import org.apache.doris.persist.AlterDatabasePropertyInfo; import org.apache.doris.persist.AutoIncrementIdUpdateLog; import org.apache.doris.persist.ColocatePersistInfo; +import org.apache.doris.persist.CreateDbInfo; import org.apache.doris.persist.DatabaseInfo; import org.apache.doris.persist.DropDbInfo; import org.apache.doris.persist.DropInfo; @@ -448,7 +449,8 @@ public void createDb(CreateDbStmt stmt) throws DdlException { } try { unprotectCreateDb(db); - Env.getCurrentEnv().getEditLog().logCreateDb(db); + CreateDbInfo dbInfo = new CreateDbInfo(InternalCatalog.INTERNAL_CATALOG_NAME, db.getName(), db); + Env.getCurrentEnv().getEditLog().logCreateDb(dbInfo); } finally { db.writeUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 11a5a58e42bf01..3d103a9ff67d7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -1419,11 +1419,7 @@ public void logSaveTransactionId(long transactionId) { logEdit(OperationType.OP_SAVE_TRANSACTION_ID, new Text(Long.toString(transactionId))); } - public void logCreateDb(Database db) { - logEdit(OperationType.OP_CREATE_DB, db); - } - - public void logNewCreateDb(CreateDbInfo info) { + public void logCreateDb(CreateDbInfo info) { logEdit(OperationType.OP_NEW_CREATE_DB, info); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 0c21533d6b25ff..8636a3dbc5c299 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -27,7 +27,8 @@ public class OperationType { // OP_LOCAL_EOF is only for local edit log, to indicate the end of a edit log run. public static final short OP_LOCAL_EOF = -1; public static final short OP_SAVE_NEXTID = 0; - public static final short OP_CREATE_DB = 1; + @Deprecated + public static final short OP_CREATE_DB = 1; // deprecated, use OP_NEW_CREATE_DB instead public static final short OP_DROP_DB = 2; public static final short OP_ALTER_DB = 3; public static final short OP_ERASE_DB = 4; diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/CreateDbInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/CreateDbInfoTest.java new file mode 100644 index 00000000000000..881c61065be093 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/CreateDbInfoTest.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist; + +import org.apache.doris.catalog.Database; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.meta.MetaContext; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.nio.file.Files; + +public class CreateDbInfoTest { + @Test + public void testSerialization() throws Exception { + MetaContext metaContext = new MetaContext(); + metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT); + metaContext.setThreadLocalInfo(); + + // 1. Write objects to file + File file = new File("./createDbInfo"); + file.createNewFile(); + DataOutputStream dos = new DataOutputStream(Files.newOutputStream(file.toPath())); + + Database db = new Database(10000, "db1"); + CreateDbInfo info1 = new CreateDbInfo(InternalCatalog.INTERNAL_CATALOG_NAME, db.getName(), db); + info1.write(dos); + + CreateDbInfo info2 = new CreateDbInfo("external_catalog", "external_db", null); + info2.write(dos); + + dos.flush(); + dos.close(); + + // 2. Read objects from file + DataInputStream dis = new DataInputStream(Files.newInputStream(file.toPath())); + + CreateDbInfo rInfo1 = CreateDbInfo.read(dis); + Assert.assertEquals(info1.getCtlName(), rInfo1.getCtlName()); + Assert.assertEquals(info1.getDbName(), rInfo1.getDbName()); + Assert.assertEquals(info1.getInternalDb().getId(), rInfo1.getInternalDb().getId()); + + CreateDbInfo rInfo2 = CreateDbInfo.read(dis); + Assert.assertEquals(info2.getCtlName(), rInfo2.getCtlName()); + Assert.assertEquals(info2.getDbName(), rInfo2.getDbName()); + Assert.assertNull(rInfo2.getInternalDb()); + + // 3. delete files + dis.close(); + file.delete(); + } +} From bcdbc9cd87f0fb8aa88b8f237b234f1a78423ae5 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 10 Jan 2025 10:47:43 +0800 Subject: [PATCH 9/9] 9 --- .../org/apache/doris/persist/CreateTableInfo.java | 9 ++++++++- .../java/org/apache/doris/persist/DropInfo.java | 14 ++++++++++++++ .../java/org/apache/doris/persist/EditLog.java | 6 ++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java index 87ad4d7ad9974c..eab2f2b872ea5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java @@ -23,9 +23,11 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; +import com.google.common.base.Strings; import com.google.gson.annotations.SerializedName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +55,7 @@ public CreateTableInfo() { // for internal table public CreateTableInfo(String dbName, Table table) { + this.ctlName = InternalCatalog.INTERNAL_CATALOG_NAME; this.dbName = dbName; this.tblName = table.getName(); this.table = table; @@ -127,7 +130,11 @@ public String toJson() { @Override public String toString() { - return toJson(); + // In previous versions, ctlName and tblName is not set, so it may be null. + return String.format("%s.%s.%s", + Strings.isNullOrEmpty(ctlName) ? InternalCatalog.INTERNAL_CATALOG_NAME : ctlName, + dbName, + Strings.isNullOrEmpty(tblName) ? table.getName() : tblName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java index a1e679a18dc99b..ff80f6ecb17aad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java @@ -21,8 +21,10 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.persist.gson.GsonUtils; +import com.google.common.base.Strings; import com.google.gson.annotations.SerializedName; import java.io.DataInput; @@ -61,13 +63,16 @@ public DropInfo(String ctl, String db, String tbl) { this.tableName = tbl; } + // for internal table public DropInfo(long dbId, long tableId, String tableName, boolean isView, boolean forceDrop, long recycleTime) { this(dbId, tableId, tableName, -1L, "", isView, forceDrop, recycleTime); } + // for internal table public DropInfo(long dbId, long tableId, String tableName, long indexId, String indexName, boolean isView, boolean forceDrop, long recycleTime) { + this.ctl = InternalCatalog.INTERNAL_CATALOG_NAME; this.dbId = dbId; this.tableId = tableId; this.tableName = tableName; @@ -168,4 +173,13 @@ public String toJson() { public static DropInfo fromJson(String json) { return GsonUtils.GSON.fromJson(json, DropInfo.class); } + + @Override + public String toString() { + // In previous versions, ctl and db are not set, so they may be null. + return String.format("%s.%s.%s", + Strings.isNullOrEmpty(ctl) ? InternalCatalog.INTERNAL_CATALOG_NAME : ctl, + Strings.isNullOrEmpty(db) ? dbId : db, + tableName); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 3d103a9ff67d7f..9aac18e7fbeec8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -234,8 +234,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { } case OperationType.OP_CREATE_TABLE: { CreateTableInfo info = (CreateTableInfo) journal.getData(); - LOG.info("Begin to unprotect create table. db = " + info.getDbName() + " table = " + info.getTable() - .getId()); + LOG.info("Begin to unprotect create table. {}", info); env.replayCreateTable(info); if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName().equals( InternalCatalog.INTERNAL_CATALOG_NAME)) { @@ -253,11 +252,10 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { } case OperationType.OP_DROP_TABLE: { DropInfo info = (DropInfo) journal.getData(); + LOG.info("Begin to unprotect drop table: {}", info); if (Strings.isNullOrEmpty(info.getCtl()) || info.getCtl().equals( InternalCatalog.INTERNAL_CATALOG_NAME)) { Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(info.getDbId()); - LOG.info("Begin to unprotect drop table. db = {} table = {}", db.getFullName(), - info.getTableId()); env.replayDropTable(db, info.getTableId(), info.isForceDrop(), info.getRecycleTime()); DropTableRecord record = new DropTableRecord(logId, info); env.getBinlogManager().addDropTableRecord(record);