Skip to content

Commit c31cc79

Browse files
kasakriszkasakrisz
committed
HIVE-29496: Subsequent transactional materialized view rebuild fails when source tables are iceberg tables
1 parent 50f4044 commit c31cc79

File tree

3 files changed

+200
-17
lines changed

3 files changed

+200
-17
lines changed

‎iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc9.q‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
-- Test subsequent rebuild of transactional MV when source table is Iceberg.
2+
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
3+
set hive.support.concurrency=true;
4+
5+
create table ice_basetable (a int, b string) stored by iceberg;
6+
insert into ice_basetable values (1, 'alfred'),(2, 'bob'),(2, 'bonnie'),(3, 'calvin'),(3, 'charlie');
7+
8+
create materialized view mv_acid STORED AS ORC TBLPROPERTIES ('transactional'='true') as
9+
select b, a from ice_basetable;
10+
11+
create materialized view mv_insert_only STORED AS ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only') as
12+
select b, a from ice_basetable;
13+
14+
select * from mv_acid;
15+
select * from mv_insert_only;
16+
17+
insert into ice_basetable values (5, 'amia');
18+
alter materialized view mv_acid rebuild;
19+
alter materialized view mv_insert_only rebuild;
20+
21+
insert into ice_basetable values (4, 'mania');
22+
alter materialized view mv_acid rebuild;
23+
alter materialized view mv_insert_only rebuild;
24+
25+
select * from mv_acid;
26+
select * from mv_insert_only;

‎iceberg/iceberg-handler/src/test/results/positive/llap/mv_iceberg_orc9.q.out‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
PREHOOK: query: create table ice_basetable (a int, b string) stored by iceberg
2+
PREHOOK: type: CREATETABLE
3+
PREHOOK: Output: database:default
4+
PREHOOK: Output: default@ice_basetable
5+
POSTHOOK: query: create table ice_basetable (a int, b string) stored by iceberg
6+
POSTHOOK: type: CREATETABLE
7+
POSTHOOK: Output: database:default
8+
POSTHOOK: Output: default@ice_basetable
9+
PREHOOK: query: insert into ice_basetable values (1, 'alfred'),(2, 'bob'),(2, 'bonnie'),(3, 'calvin'),(3, 'charlie')
10+
PREHOOK: type: QUERY
11+
PREHOOK: Input: _dummy_database@_dummy_table
12+
PREHOOK: Output: default@ice_basetable
13+
POSTHOOK: query: insert into ice_basetable values (1, 'alfred'),(2, 'bob'),(2, 'bonnie'),(3, 'calvin'),(3, 'charlie')
14+
POSTHOOK: type: QUERY
15+
POSTHOOK: Input: _dummy_database@_dummy_table
16+
POSTHOOK: Output: default@ice_basetable
17+
PREHOOK: query: create materialized view mv_acid STORED AS ORC TBLPROPERTIES ('transactional'='true') as
18+
select b, a from ice_basetable
19+
PREHOOK: type: CREATE_MATERIALIZED_VIEW
20+
PREHOOK: Input: default@ice_basetable
21+
PREHOOK: Output: database:default
22+
PREHOOK: Output: default@mv_acid
23+
POSTHOOK: query: create materialized view mv_acid STORED AS ORC TBLPROPERTIES ('transactional'='true') as
24+
select b, a from ice_basetable
25+
POSTHOOK: type: CREATE_MATERIALIZED_VIEW
26+
POSTHOOK: Input: default@ice_basetable
27+
POSTHOOK: Output: database:default
28+
POSTHOOK: Output: default@mv_acid
29+
POSTHOOK: Lineage: mv_acid.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
30+
POSTHOOK: Lineage: mv_acid.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
31+
PREHOOK: query: create materialized view mv_insert_only STORED AS ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only') as
32+
select b, a from ice_basetable
33+
PREHOOK: type: CREATE_MATERIALIZED_VIEW
34+
PREHOOK: Input: default@ice_basetable
35+
PREHOOK: Output: database:default
36+
PREHOOK: Output: default@mv_insert_only
37+
POSTHOOK: query: create materialized view mv_insert_only STORED AS ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only') as
38+
select b, a from ice_basetable
39+
POSTHOOK: type: CREATE_MATERIALIZED_VIEW
40+
POSTHOOK: Input: default@ice_basetable
41+
POSTHOOK: Output: database:default
42+
POSTHOOK: Output: default@mv_insert_only
43+
POSTHOOK: Lineage: mv_insert_only.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
44+
POSTHOOK: Lineage: mv_insert_only.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
45+
PREHOOK: query: select * from mv_acid
46+
PREHOOK: type: QUERY
47+
PREHOOK: Input: default@mv_acid
48+
#### A masked pattern was here ####
49+
POSTHOOK: query: select * from mv_acid
50+
POSTHOOK: type: QUERY
51+
POSTHOOK: Input: default@mv_acid
52+
#### A masked pattern was here ####
53+
alfred 1
54+
bob 2
55+
bonnie 2
56+
calvin 3
57+
charlie 3
58+
PREHOOK: query: select * from mv_insert_only
59+
PREHOOK: type: QUERY
60+
PREHOOK: Input: default@mv_insert_only
61+
#### A masked pattern was here ####
62+
POSTHOOK: query: select * from mv_insert_only
63+
POSTHOOK: type: QUERY
64+
POSTHOOK: Input: default@mv_insert_only
65+
#### A masked pattern was here ####
66+
alfred 1
67+
bob 2
68+
bonnie 2
69+
calvin 3
70+
charlie 3
71+
PREHOOK: query: insert into ice_basetable values (5, 'amia')
72+
PREHOOK: type: QUERY
73+
PREHOOK: Input: _dummy_database@_dummy_table
74+
PREHOOK: Output: default@ice_basetable
75+
POSTHOOK: query: insert into ice_basetable values (5, 'amia')
76+
POSTHOOK: type: QUERY
77+
POSTHOOK: Input: _dummy_database@_dummy_table
78+
POSTHOOK: Output: default@ice_basetable
79+
PREHOOK: query: alter materialized view mv_acid rebuild
80+
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
81+
PREHOOK: Input: default@ice_basetable
82+
PREHOOK: Output: default@mv_acid
83+
POSTHOOK: query: alter materialized view mv_acid rebuild
84+
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
85+
POSTHOOK: Input: default@ice_basetable
86+
POSTHOOK: Output: default@mv_acid
87+
POSTHOOK: Lineage: mv_acid.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
88+
POSTHOOK: Lineage: mv_acid.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
89+
PREHOOK: query: alter materialized view mv_insert_only rebuild
90+
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
91+
PREHOOK: Input: default@ice_basetable
92+
PREHOOK: Output: default@mv_insert_only
93+
POSTHOOK: query: alter materialized view mv_insert_only rebuild
94+
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
95+
POSTHOOK: Input: default@ice_basetable
96+
POSTHOOK: Output: default@mv_insert_only
97+
POSTHOOK: Lineage: mv_insert_only.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
98+
POSTHOOK: Lineage: mv_insert_only.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
99+
PREHOOK: query: insert into ice_basetable values (4, 'mania')
100+
PREHOOK: type: QUERY
101+
PREHOOK: Input: _dummy_database@_dummy_table
102+
PREHOOK: Output: default@ice_basetable
103+
POSTHOOK: query: insert into ice_basetable values (4, 'mania')
104+
POSTHOOK: type: QUERY
105+
POSTHOOK: Input: _dummy_database@_dummy_table
106+
POSTHOOK: Output: default@ice_basetable
107+
PREHOOK: query: alter materialized view mv_acid rebuild
108+
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
109+
PREHOOK: Input: default@ice_basetable
110+
PREHOOK: Output: default@mv_acid
111+
POSTHOOK: query: alter materialized view mv_acid rebuild
112+
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
113+
POSTHOOK: Input: default@ice_basetable
114+
POSTHOOK: Output: default@mv_acid
115+
POSTHOOK: Lineage: mv_acid.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
116+
POSTHOOK: Lineage: mv_acid.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
117+
PREHOOK: query: alter materialized view mv_insert_only rebuild
118+
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
119+
PREHOOK: Input: default@ice_basetable
120+
PREHOOK: Output: default@mv_insert_only
121+
POSTHOOK: query: alter materialized view mv_insert_only rebuild
122+
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
123+
POSTHOOK: Input: default@ice_basetable
124+
POSTHOOK: Output: default@mv_insert_only
125+
POSTHOOK: Lineage: mv_insert_only.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
126+
POSTHOOK: Lineage: mv_insert_only.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
127+
PREHOOK: query: select * from mv_acid
128+
PREHOOK: type: QUERY
129+
PREHOOK: Input: default@mv_acid
130+
#### A masked pattern was here ####
131+
POSTHOOK: query: select * from mv_acid
132+
POSTHOOK: type: QUERY
133+
POSTHOOK: Input: default@mv_acid
134+
#### A masked pattern was here ####
135+
alfred 1
136+
bob 2
137+
bonnie 2
138+
calvin 3
139+
charlie 3
140+
mania 4
141+
amia 5
142+
PREHOOK: query: select * from mv_insert_only
143+
PREHOOK: type: QUERY
144+
PREHOOK: Input: default@mv_insert_only
145+
#### A masked pattern was here ####
146+
POSTHOOK: query: select * from mv_insert_only
147+
POSTHOOK: type: QUERY
148+
POSTHOOK: Input: default@mv_insert_only
149+
#### A masked pattern was here ####
150+
alfred 1
151+
bob 2
152+
bonnie 2
153+
calvin 3
154+
charlie 3
155+
mania 4
156+
amia 5

‎ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,24 @@ public void analyzeInternal(ASTNode root) throws SemanticException {
179179

180180
LOG.debug("Rebuilding materialized view " + tableName.getNotEmptyDbTable());
181181
super.analyzeInternal(rewrittenAST);
182+
183+
if (!this.ctx.isExplainPlan() && AcidUtils.isTransactionalTable(mvTable)) {
184+
// Acquire lock for the given materialized view. Only one rebuild per materialized view can be triggered at a
185+
// given time, as otherwise we might produce incorrect results if incremental maintenance is triggered.
186+
HiveTxnManager txnManager = getTxnMgr();
187+
LockState state;
188+
try {
189+
state = txnManager.acquireMaterializationRebuildLock(new LockMaterializationRebuildRequest(tableName.getCat(),
190+
tableName.getDb(), tableName.getTable(), txnManager.getCurrentTxnId())).getState();
191+
} catch (LockException e) {
192+
throw new SemanticException("Exception acquiring lock for rebuilding the materialized view", e);
193+
}
194+
if (state != LockState.ACQUIRED) {
195+
throw new SemanticException(
196+
"Another process is rebuilding the materialized view " + tableName.getNotEmptyDbTable());
197+
}
198+
}
199+
182200
queryState.setCommandType(HiveOperation.ALTER_MATERIALIZED_VIEW_REBUILD);
183201
}
184202

@@ -206,23 +224,6 @@ private ASTNode getRewrittenAST(TableName tableName) throws SemanticException {
206224
tableName.getEscapedNotEmptyDbTable(), viewText);
207225
rewrittenAST = ParseUtils.parse(rewrittenInsertStatement, ctx);
208226
this.ctx.addSubContext(ctx);
209-
210-
if (!this.ctx.isExplainPlan() && AcidUtils.isTransactionalTable(table)) {
211-
// Acquire lock for the given materialized view. Only one rebuild per materialized view can be triggered at a
212-
// given time, as otherwise we might produce incorrect results if incremental maintenance is triggered.
213-
HiveTxnManager txnManager = getTxnMgr();
214-
LockState state;
215-
try {
216-
state = txnManager.acquireMaterializationRebuildLock(new LockMaterializationRebuildRequest(tableName.getCat(),
217-
tableName.getDb(), tableName.getTable(), txnManager.getCurrentTxnId())).getState();
218-
} catch (LockException e) {
219-
throw new SemanticException("Exception acquiring lock for rebuilding the materialized view", e);
220-
}
221-
if (state != LockState.ACQUIRED) {
222-
throw new SemanticException(
223-
"Another process is rebuilding the materialized view " + tableName.getNotEmptyDbTable());
224-
}
225-
}
226227
} catch (Exception e) {
227228
throw new SemanticException(e);
228229
}

0 commit comments

Comments
 (0)