mirrors
/
redis
mirror of https://mirror.osredm.com/root/redis.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Trim rdb loading code for pre-release formats (#11058) 

The initial module format introduced in 4.0  RC1 and was changed in RC2
The initial function format introduced in 7.0 RC1 and changed in RC3
This commit is contained in:
Oran Agra 2022-08-15 21:41:44 +03:00 committed by GitHub
parent 1189680edd
commit ac1cc5a6e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 52 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/functions.c
View File

@ -754,11 +754,15 @@ void functionRestoreCommand(client *c) {
err = sdsnew("can not read data type"); err = sdsnew("can not read data type");
goto load_error; goto load_error;
} }
if (type != RDB_OPCODE_FUNCTION && type != RDB_OPCODE_FUNCTION2) { if (type == RDB_OPCODE_FUNCTION_PRE_GA) {
err = sdsnew("Pre-GA function format not supported");
goto load_error;
}
if (type != RDB_OPCODE_FUNCTION2) {
err = sdsnew("given type is not a function"); err = sdsnew("given type is not a function");
goto load_error; goto load_error;
} }
if (rdbFunctionLoad(&payload, rdbver, functions_lib_ctx, type, RDBFLAGS_NONE, &err) != C_OK) { if (rdbFunctionLoad(&payload, rdbver, functions_lib_ctx, RDBFLAGS_NONE, &err) != C_OK) {
if (!err) { if (!err) {
err = sdsnew("failed loading the given functions payload"); err = sdsnew("failed loading the given functions payload");
} }

25
src/module.c
View File

@ -6543,10 +6543,8 @@ saveerr:
* new data types. */ * new data types. */
uint64_t RM_LoadUnsigned(RedisModuleIO *io) { uint64_t RM_LoadUnsigned(RedisModuleIO *io) {
if (io->error) return 0; if (io->error) return 0;
if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL);
uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr;
if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr;
}
uint64_t value; uint64_t value;
int retval = rdbLoadLenByRef(io->rio, NULL, &value); int retval = rdbLoadLenByRef(io->rio, NULL, &value);
if (retval == -1) goto loaderr; if (retval == -1) goto loaderr;
@ -6614,10 +6612,8 @@ saveerr:
/* Implements RM_LoadString() and RM_LoadStringBuffer() */ /* Implements RM_LoadString() and RM_LoadStringBuffer() */
void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) {
if (io->error) return NULL; if (io->error) return NULL;
if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL);
uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr;
if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr;
}
void *s = rdbGenericLoadStringObject(io->rio, void *s = rdbGenericLoadStringObject(io->rio,
plain ? RDB_LOAD_PLAIN : RDB_LOAD_NONE, lenptr); plain ? RDB_LOAD_PLAIN : RDB_LOAD_NONE, lenptr);
if (s == NULL) goto loaderr; if (s == NULL) goto loaderr;
@ -6675,10 +6671,8 @@ saveerr:
* double value saved by RedisModule_SaveDouble(). */ * double value saved by RedisModule_SaveDouble(). */
double RM_LoadDouble(RedisModuleIO *io) { double RM_LoadDouble(RedisModuleIO *io) {
if (io->error) return 0; if (io->error) return 0;
if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL);
uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr;
if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr;
}
double value; double value;
int retval = rdbLoadBinaryDoubleValue(io->rio, &value); int retval = rdbLoadBinaryDoubleValue(io->rio, &value);
if (retval == -1) goto loaderr; if (retval == -1) goto loaderr;
@ -6712,10 +6706,8 @@ saveerr:
* float value saved by RedisModule_SaveFloat(). */ * float value saved by RedisModule_SaveFloat(). */
float RM_LoadFloat(RedisModuleIO *io) { float RM_LoadFloat(RedisModuleIO *io) {
if (io->error) return 0; if (io->error) return 0;
if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL);
uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr;
if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr;
}
float value; float value;
int retval = rdbLoadBinaryFloatValue(io->rio, &value); int retval = rdbLoadBinaryFloatValue(io->rio, &value);
if (retval == -1) goto loaderr; if (retval == -1) goto loaderr;
@ -6866,7 +6858,6 @@ void *RM_LoadDataTypeFromStringEncver(const RedisModuleString *str, const module
/* All RM_Save*() calls always write a version 2 compatible format, so we /* All RM_Save*() calls always write a version 2 compatible format, so we
* need to make sure we read the same. * need to make sure we read the same.
*/ */
io.ver = 2;
ret = mt->rdb_load(&io,encver); ret = mt->rdb_load(&io,encver);
if (io.ctx) { if (io.ctx) {
moduleFreeContext(io.ctx); moduleFreeContext(io.ctx);

102
src/rdb.c
View File

@ -2605,7 +2605,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
raxStop(&ri_cg_pel); raxStop(&ri_cg_pel);
} }
} }
} else if (rdbtype == RDB_TYPE_MODULE || rdbtype == RDB_TYPE_MODULE_2) { } else if (rdbtype == RDB_TYPE_MODULE_PRE_GA) {
rdbReportCorruptRDB("Pre-release module format not supported");
return NULL;
} else if (rdbtype == RDB_TYPE_MODULE_2) {
uint64_t moduleid = rdbLoadLen(rdb,NULL); uint64_t moduleid = rdbLoadLen(rdb,NULL);
if (rioGetReadError(rdb)) { if (rioGetReadError(rdb)) {
rdbReportReadError("Short read module id"); rdbReportReadError("Short read module id");
@ -2613,7 +2616,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
} }
moduleType *mt = moduleTypeLookupModuleByID(moduleid); moduleType *mt = moduleTypeLookupModuleByID(moduleid);
if (rdbCheckMode && rdbtype == RDB_TYPE_MODULE_2) { if (rdbCheckMode) {
char name[10]; char name[10];
moduleTypeNameByID(name,moduleid); moduleTypeNameByID(name,moduleid);
return rdbLoadCheckModuleValue(rdb,name); return rdbLoadCheckModuleValue(rdb,name);
@ -2629,7 +2632,6 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
robj keyobj; robj keyobj;
initStaticStringObject(keyobj,key); initStaticStringObject(keyobj,key);
moduleInitIOContext(io,mt,rdb,&keyobj,dbid); moduleInitIOContext(io,mt,rdb,&keyobj,dbid);
io.ver = (rdbtype == RDB_TYPE_MODULE) ? 1 : 2;
/* Call the rdb_load method of the module providing the 10 bit /* Call the rdb_load method of the module providing the 10 bit
* encoding version in the lower 10 bits of the module ID. */ * encoding version in the lower 10 bits of the module ID. */
void *ptr = mt->rdb_load(&io,moduleid&1023); void *ptr = mt->rdb_load(&io,moduleid&1023);
@ -2639,24 +2641,22 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
} }
/* Module v2 serialization has an EOF mark at the end. */ /* Module v2 serialization has an EOF mark at the end. */
if (io.ver == 2) { uint64_t eof = rdbLoadLen(rdb,NULL);
uint64_t eof = rdbLoadLen(rdb,NULL); if (eof == RDB_LENERR) {
if (eof == RDB_LENERR) { if (ptr) {
if (ptr) { o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */ decrRefCount(o);
decrRefCount(o);
}
return NULL;
} }
if (eof != RDB_MODULE_OPCODE_EOF) { return NULL;
rdbReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by " }
"the proper module value EOF marker", moduleTypeModuleName(mt)); if (eof != RDB_MODULE_OPCODE_EOF) {
if (ptr) { rdbReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by "
o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */ "the proper module value EOF marker", moduleTypeModuleName(mt));
decrRefCount(o); if (ptr) {
} o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
return NULL; decrRefCount(o);
} }
return NULL;
} }
if (ptr == NULL) { if (ptr == NULL) {
@ -2794,62 +2794,14 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) {
* *
* The lib_ctx argument is also optional. If NULL is given, only verify rdb * The lib_ctx argument is also optional. If NULL is given, only verify rdb
* structure with out performing the actual functions loading. */ * structure with out performing the actual functions loading. */
int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int type, int rdbflags, sds *err) { int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int rdbflags, sds *err) {
UNUSED(ver); UNUSED(ver);
sds error = NULL; sds error = NULL;
sds final_payload = NULL; sds final_payload = NULL;
int res = C_ERR; int res = C_ERR;
if (type == RDB_OPCODE_FUNCTION) { if (!(final_payload = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
/* RDB that was generated on versions 7.0 rc1 and 7.0 rc2 has another error = sdsnew("Failed loading library payload");
* an old format that contains the library name, engine and description. goto done;
* To support this format we must read those values. */
sds name = NULL;
sds engine_name = NULL;
sds desc = NULL;
sds blob = NULL;
uint64_t has_desc;
if (!(name = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading library name");
goto cleanup;
}
if (!(engine_name = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading engine name");
goto cleanup;
}
if ((has_desc = rdbLoadLen(rdb, NULL)) == RDB_LENERR) {
error = sdsnew("Failed loading library description indicator");
goto cleanup;
}
if (has_desc && !(desc = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading library description");
goto cleanup;
}
if (!(blob = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading library blob");
goto cleanup;
}
/* Translate old format (versions 7.0 rc1 and 7.0 rc2) to new format.
* The new format has the library name and engine inside the script payload.
* Add those parameters to the original script payload (ignore the description if exists). */
final_payload = sdscatfmt(sdsempty(), "#!%s name=%s\n%s", engine_name, name, blob);
cleanup:
if (name) sdsfree(name);
if (engine_name) sdsfree(engine_name);
if (desc) sdsfree(desc);
if (blob) sdsfree(blob);
if (error) goto done;
} else if (type == RDB_OPCODE_FUNCTION2) {
if (!(final_payload = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading library payload");
goto done;
}
} else {
serverPanic("Bad function type was given to rdbFunctionLoad");
} }
if (lib_ctx) { if (lib_ctx) {
@ -3070,7 +3022,6 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin
RedisModuleIO io; RedisModuleIO io;
moduleInitIOContext(io,mt,rdb,NULL,-1); moduleInitIOContext(io,mt,rdb,NULL,-1);
io.ver = 2;
/* Call the rdb_load method of the module providing the 10 bit /* Call the rdb_load method of the module providing the 10 bit
* encoding version in the lower 10 bits of the module ID. */ * encoding version in the lower 10 bits of the module ID. */
int rc = mt->aux_load(&io,moduleid&1023, when); int rc = mt->aux_load(&io,moduleid&1023, when);
@ -3095,9 +3046,12 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin
decrRefCount(aux); decrRefCount(aux);
continue; /* Read next opcode. */ continue; /* Read next opcode. */
} }
} else if (type == RDB_OPCODE_FUNCTION || type == RDB_OPCODE_FUNCTION2) { } else if (type == RDB_OPCODE_FUNCTION_PRE_GA) {
rdbReportCorruptRDB("Pre-release function format not supported.");
exit(1);
} else if (type == RDB_OPCODE_FUNCTION2) {
sds err = NULL; sds err = NULL;
if (rdbFunctionLoad(rdb, rdbver, rdb_loading_ctx->functions_lib_ctx, type, rdbflags, &err) != C_OK) { if (rdbFunctionLoad(rdb, rdbver, rdb_loading_ctx->functions_lib_ctx, rdbflags, &err) != C_OK) {
serverLog(LL_WARNING,"Failed loading library, %s", err); serverLog(LL_WARNING,"Failed loading library, %s", err);
sdsfree(err); sdsfree(err);
goto eoferr; goto eoferr;

6
src/rdb.h
View File

@ -78,7 +78,7 @@
#define RDB_TYPE_ZSET 3 #define RDB_TYPE_ZSET 3
#define RDB_TYPE_HASH 4 #define RDB_TYPE_HASH 4
#define RDB_TYPE_ZSET_2 5 /* ZSET version 2 with doubles stored in binary. */ #define RDB_TYPE_ZSET_2 5 /* ZSET version 2 with doubles stored in binary. */
#define RDB_TYPE_MODULE 6 #define RDB_TYPE_MODULE_PRE_GA 6 /* Used in 4.0 release candidates */
#define RDB_TYPE_MODULE_2 7 /* Module value with annotations for parsing without #define RDB_TYPE_MODULE_2 7 /* Module value with annotations for parsing without
the generating module being loaded. */ the generating module being loaded. */
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
@ -102,7 +102,7 @@
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
#define RDB_OPCODE_FUNCTION2 245 /* function library data */ #define RDB_OPCODE_FUNCTION2 245 /* function library data */
#define RDB_OPCODE_FUNCTION 246 /* old function library data for 7.0 rc1 and rc2 */ #define RDB_OPCODE_FUNCTION_PRE_GA 246 /* old function library data for 7.0 rc1 and rc2 */
#define RDB_OPCODE_MODULE_AUX 247 /* Module auxiliary data. */ #define RDB_OPCODE_MODULE_AUX 247 /* Module auxiliary data. */
#define RDB_OPCODE_IDLE 248 /* LRU idle time. */ #define RDB_OPCODE_IDLE 248 /* LRU idle time. */
#define RDB_OPCODE_FREQ 249 /* LFU frequency. */ #define RDB_OPCODE_FREQ 249 /* LFU frequency. */
@ -171,7 +171,7 @@ int rdbSaveBinaryFloatValue(rio *rdb, float val);
int rdbLoadBinaryFloatValue(rio *rdb, float *val); int rdbLoadBinaryFloatValue(rio *rdb, float *val);
int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi); int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi);
int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadingCtx *rdb_loading_ctx); int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadingCtx *rdb_loading_ctx);
int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int type, int rdbflags, sds *err); int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int rdbflags, sds *err);
int rdbSaveRio(int req, rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi); int rdbSaveRio(int req, rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi);
ssize_t rdbSaveFunctions(rio *rdb); ssize_t rdbSaveFunctions(rio *rdb);
rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi); rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi);

10
src/redis-check-rdb.c
View File

@ -85,8 +85,9 @@ char *rdb_type_string[] = {
"zset-v1", "zset-v1",
"hash-hashtable", "hash-hashtable",
"zset-v2", "zset-v2",
"module-pre-release",
"module-value", "module-value",
"","", "",
"hash-zipmap", "hash-zipmap",
"list-ziplist", "list-ziplist",
"set-intset", "set-intset",
@ -312,10 +313,13 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) {
robj *o = rdbLoadCheckModuleValue(&rdb,name); robj *o = rdbLoadCheckModuleValue(&rdb,name);
decrRefCount(o); decrRefCount(o);
continue; /* Read type again. */ continue; /* Read type again. */
} else if (type == RDB_OPCODE_FUNCTION || type == RDB_OPCODE_FUNCTION2) { } else if (type == RDB_OPCODE_FUNCTION_PRE_GA) {
rdbCheckError("Pre-release function format not supported %d",rdbver);
goto err;
} else if (type == RDB_OPCODE_FUNCTION2) {
sds err = NULL; sds err = NULL;
rdbstate.doing = RDB_CHECK_DOING_READ_FUNCTIONS; rdbstate.doing = RDB_CHECK_DOING_READ_FUNCTIONS;
if (rdbFunctionLoad(&rdb, rdbver, NULL, type, 0, &err) != C_OK) { if (rdbFunctionLoad(&rdb, rdbver, NULL, 0, &err) != C_OK) {
rdbCheckError("Failed loading library, %s", err); rdbCheckError("Failed loading library, %s", err);
sdsfree(err); sdsfree(err);
goto err; goto err;

3
src/server.h
View File

@ -791,8 +791,6 @@ typedef struct RedisModuleIO {
rio *rio; /* Rio stream. */ rio *rio; /* Rio stream. */
moduleType *type; /* Module type doing the operation. */ moduleType *type; /* Module type doing the operation. */
int error; /* True if error condition happened. */ int error; /* True if error condition happened. */
int ver; /* Module serialization version: 1 (old),
* 2 (current version with opcodes annotation). */
struct RedisModuleCtx *ctx; /* Optional context, see RM_GetContextFromIO()*/ struct RedisModuleCtx *ctx; /* Optional context, see RM_GetContextFromIO()*/
struct redisObject *key; /* Optional name of key processed */ struct redisObject *key; /* Optional name of key processed */
int dbid; /* The dbid of the key being processed, -1 when unknown. */ int dbid; /* The dbid of the key being processed, -1 when unknown. */
@ -805,7 +803,6 @@ typedef struct RedisModuleIO {
iovar.type = mtype; \ iovar.type = mtype; \
iovar.bytes = 0; \ iovar.bytes = 0; \
iovar.error = 0; \ iovar.error = 0; \
iovar.ver = 0; \
iovar.key = keyptr; \ iovar.key = keyptr; \
iovar.dbid = db; \ iovar.dbid = db; \
iovar.ctx = NULL; \ iovar.ctx = NULL; \