feature: mongo 增加 bulkWrite (#2058)

mongo8.0 新增跨集合更新api: bulkWrite(相对单一更新 性能大幅提升)
btw: 低量级数据建议继续选择单一更新(insert/update等 )

应用场景:
1. 聚合数据更新
2. 日志db跨集合写入
This commit is contained in:
云华
2025-06-05 19:52:24 +08:00
committed by GitHub
parent 5885514138
commit 4ff4db3161

View File

@@ -88,6 +88,38 @@ local function __parse_addr(addr)
return host, tonumber(port) return host, tonumber(port)
end end
local function werror(r)
local ok = (r.ok == 1 and not r.writeErrors and not r.writeConcernError and not r.errmsg)
local err
if not ok then
if r.writeErrors then
err = r.writeErrors[1].errmsg
elseif r.writeConcernError then
err = r.writeConcernError.errmsg
else
err = r.errmsg
end
end
return ok, err, r
end
local function wbulkerror(r)
local ok, err = werror(r)
local cursor
if (r.nErrors and r.nErrors ~= 0) and r.cursor then
cursor = {}
err = err and err .. ";" or ""
for i, v in ipairs(r.cursor.firstBatch) do
if v.ok ~= 1 then
cursor[#cursor+1] = i
err = err .. v.errmsg .. ";"
end
end
end
return ok, err, r, cursor
end
local auth_method = {} local auth_method = {}
local function mongo_auth(mongoc) local function mongo_auth(mongoc)
@@ -197,6 +229,61 @@ function mongo_client:runCommand(...)
return self.admin:runCommand(...) return self.admin:runCommand(...)
end end
local function filter(tbl, key, value)
if value == nil then return end
tbl[#tbl+1] = key
tbl[#tbl+1] = type(value) == "table" and bson_encode(value) or value
end
local bulkWrite = {}
bulkWrite.insert = function(nsindex, doc)
return bson_encode_order("insert", nsindex, "document", bson_encode(doc.insert))
end
bulkWrite.delete = function(nsindex, doc)
local args = {"delete", nsindex, "filter", bson_encode(doc.filter), "multi", doc.multi or false}
filter(args, "hit", doc.hit)
filter(args, "collation", doc.collation)
return bson_encode_order(table.unpack(args))
end
--update仅支持原子操作更新/聚合管道更新 详情阅读官方文档
--arrayFilters 过滤方式需要遍历encode 暂时不支持
bulkWrite.update = function(nsindex, doc)
local args = {"update", nsindex, "filter", bson_encode(doc.filter), "updateMods", bson_encode(doc.update),
"multi", doc.multi or false, "upsert", doc.upsert or false}
filter(args, "hit", doc.hit)
filter(args, "constants", doc.constants)
filter(args, "collation", doc.collation)
return bson_encode_order(table.unpack(args))
end
--bulkWrite 依赖 mongo8.0x
--忽略let,cursor,writeConcern 参数
---@param datas table 待写入数据
--- - op string 写入数据方式 参见bulkWrite
--- - collection string 数据写入目标, 用于构建nsInfo 注意:采用 空间.集合方式
--- - ... 操作具体参数 insert采用insert填充document, update采用 update 填充updateMods
--- 示例:{op = "insert", collection = "log.online", insert = {count=0}}
--- {op = "update", collection = "log.online", update = {["$set"] = {count = 2}}, upsert = true}}
function mongo_client:bulkWrite(datas, comment, ordered, verify, errorsOnly)
local ops, ns, map = {}, {}, {}
for i, v in ipairs(datas) do
if not map[v.collection] then
map[v.collection] = #ns --ops 索引从0开始
ns[#ns+1] = bson_encode({ns = v.collection})
end
local f = assert(bulkWrite[v.op], v.op)
ops[i] = f(map[v.collection], v)
end
local args = {"bulkWrite", 1, "ops", ops, "nsInfo", ns}
filter(args, "ordered", ordered) --是否有序执行 默认有序
filter(args, "bypassDocumentValidation", verify) --是否验证 默认验证
filter(args, "comment", comment) --日志跟踪注释
filter(args, "errorsOnly",errorsOnly) -- 仅返回错误信息
return wbulkerror(self:runCommand(table.unpack(args)))
end
function auth_method:auth_mongodb_cr(user,password) function auth_method:auth_mongodb_cr(user,password)
local password = md5.sumhexa(string.format("%s:mongo:%s",user,password)) local password = md5.sumhexa(string.format("%s:mongo:%s",user,password))
local result= self:runCommand "getnonce" local result= self:runCommand "getnonce"
@@ -355,22 +442,6 @@ function mongo_collection:insert(doc)
self.database:send_command("insert", self.name, "documents", {bson_encode(doc)}) self.database:send_command("insert", self.name, "documents", {bson_encode(doc)})
end end
local function werror(r)
local ok = (r.ok == 1 and not r.writeErrors and not r.writeConcernError and not r.errmsg)
local err
if not ok then
if r.writeErrors then
err = r.writeErrors[1].errmsg
elseif r.writeConcernError then
err = r.writeConcernError.errmsg
else
err = r.errmsg
end
end
return ok, err, r
end
function mongo_collection:safe_insert(doc) function mongo_collection:safe_insert(doc)
local r = self.database:runCommand("insert", self.name, "documents", {bson_encode(doc)}) local r = self.database:runCommand("insert", self.name, "documents", {bson_encode(doc)})
return werror(r) return werror(r)