diff --git a/src/utils/ddb/ddb_vos.c b/src/utils/ddb/ddb_vos.c index 924a5343d76..53a97978407 100644 --- a/src/utils/ddb/ddb_vos.c +++ b/src/utils/ddb/ddb_vos.c @@ -1099,7 +1099,6 @@ dump_csum_sv(daos_handle_t coh, daos_key_t *dkey, daos_unit_oid_t *oid, daos_iod { daos_handle_t ioh; struct dcs_ci_list *cil; - int cb_rc = 0; int rc; rc = vos_fetch_begin(coh, *oid, epoch, dkey, 1, iod, VOS_OF_FETCH_CSUM, NULL, &ioh, NULL); @@ -1110,19 +1109,13 @@ dump_csum_sv(daos_handle_t coh, daos_key_t *dkey, daos_unit_oid_t *oid, daos_iod } cil = vos_ioh2ci(ioh); - cb_rc = dump_cb(cb_arg, NULL, vos_ioh2sv_epoch(ioh), cil); - if (!SUCCESS(cb_rc)) + rc = dump_cb(cb_arg, NULL, vos_ioh2sv_epoch(ioh), cil); + if (!SUCCESS(rc)) D_DEBUG(DB_IO, "Csum dump callback for " DF_UOID " returned: " DF_RC "\n", - DP_UOID(*oid), DP_RC(cb_rc)); - - rc = vos_fetch_end(ioh, NULL, cb_rc); - if (!SUCCESS(rc) && rc != cb_rc) - D_ERROR("vos_fetch_end for csum dump of " DF_UOID " failed: " DF_RC "\n", DP_UOID(*oid), DP_RC(rc)); + rc = vos_fetch_end(ioh, NULL, rc); out: - if (!SUCCESS(cb_rc)) - rc = cb_rc; return rc; } @@ -1133,7 +1126,6 @@ dump_csum_recx(daos_handle_t coh, daos_key_t *dkey, daos_unit_oid_t *oid, daos_i daos_handle_t ioh; struct dcs_ci_list *cil; struct daos_recx_ep_list *rel; - int cb_rc = 0; int rc; rc = vos_fetch_begin(coh, *oid, epoch, dkey, 1, iod, VOS_OF_FETCH_CSUM, NULL, &ioh, NULL); @@ -1145,21 +1137,15 @@ dump_csum_recx(daos_handle_t coh, daos_key_t *dkey, daos_unit_oid_t *oid, daos_i cil = vos_ioh2ci(ioh); rel = vos_ioh2recx_list(ioh); - cb_rc = dump_cb(cb_arg, rel, 0, cil); - if (!SUCCESS(cb_rc)) + rc = dump_cb(cb_arg, rel, 0, cil); + if (!SUCCESS(rc)) D_DEBUG(DB_IO, "Csum dump callback for " DF_UOID " returned: " DF_RC "\n", - DP_UOID(*oid), DP_RC(cb_rc)); + DP_UOID(*oid), DP_RC(rc)); /* rel ownership is transferred by vos_ioh2recx_list(); free before vos_fetch_end. */ daos_recx_ep_list_free(rel, iod->iod_nr); - rc = vos_fetch_end(ioh, NULL, cb_rc); - if (!SUCCESS(rc) && rc != cb_rc) - D_ERROR("vos_fetch_end for csum dump of " DF_UOID " failed: " DF_RC "\n", - DP_UOID(*oid), DP_RC(rc)); - + rc = vos_fetch_end(ioh, NULL, rc); out: - if (!SUCCESS(cb_rc)) - rc = cb_rc; return rc; } @@ -1202,6 +1188,314 @@ dv_dump_csum(daos_handle_t poh, struct dv_tree_path *path, daos_epoch_t epoch, return rc; } +/* + * Fetch the record data for a single checksum-verification segment (either the whole single + * value, or one physical recx/epoch segment of an array value) into a freshly allocated + * buffer in \a sgl. Uses the same 2-pass vos_obj_fetch() pattern as dv_dump_value(): the first + * pass discovers the record size, the second pass fetches the data. + * + * A zero-length result (hole/punch at \a epoch) is not an error; the caller's sgl is left + * empty in that case. + */ +static int +fetch_check_value(daos_handle_t coh, daos_unit_oid_t *oid, daos_key_t *dkey, daos_iod_t *iod, + daos_epoch_t epoch, d_sg_list_t *sgl) +{ + size_t data_size; + int rc; + + iod->iod_size = 0; + rc = vos_obj_fetch(coh, *oid, epoch, 0, dkey, 1, iod, NULL); + if (!SUCCESS(rc)) { + D_ERROR("vos_obj_fetch (size) for csum check of " DF_UOID " failed: " DF_RC "\n", + DP_UOID(*oid), DP_RC(rc)); + return rc; + } + + data_size = iod->iod_size; + if (iod->iod_type == DAOS_IOD_ARRAY) + data_size *= iod->iod_recxs[0].rx_nr; + + if (data_size == 0) /* hole/punch: nothing to verify */ + return 0; + + D_ALLOC(sgl->sg_iovs[0].iov_buf, data_size); + if (sgl->sg_iovs[0].iov_buf == NULL) + return -DER_NOMEM; + sgl->sg_iovs[0].iov_buf_len = data_size; + + rc = vos_obj_fetch(coh, *oid, epoch, 0, dkey, 1, iod, sgl); + if (!SUCCESS(rc)) { + D_ERROR("vos_obj_fetch (data) for csum check of " DF_UOID " failed: " DF_RC "\n", + DP_UOID(*oid), DP_RC(rc)); + D_FREE(sgl->sg_iovs[0].iov_buf); + return rc; + } + + return 0; +} + +/* + * Create an independent copy of \a src (metadata + checksum bytes) in a single D_ALLOC'd + * struct+buffer, matching daos_csummer_calc_key()'s allocation pattern. The result is freed by + * the caller with a plain D_FREE() -- it is not a daos_csummer-owned allocation, so + * daos_csummer_free_ci()/_ic() must not be used on it. + */ +static int +snapshot_csum_info(struct dcs_csum_info *src, struct dcs_csum_info **snapshot) +{ + struct dcs_csum_info *dst; + uint8_t *buf; + + D_ALLOC(dst, sizeof(*dst) + src->cs_buf_len); + if (dst == NULL) + return -DER_NOMEM; + + /* &dst[1] is "one past the struct", i.e. the start of the trailing buffer allocated + * above (struct dcs_csum_info has no flexible array member of its own). */ + buf = (uint8_t *)&dst[1]; + memcpy(buf, ci_idx2csum(src, 0), src->cs_buf_len); + ci_set(dst, buf, src->cs_buf_len, src->cs_len, src->cs_nr, src->cs_chunksize, src->cs_type); + + *snapshot = dst; + return 0; +} + +/* + * Recompute the checksum of one segment's currently stored data and compare it against the + * checksum info already fetched from disk (\a ci). Returns 0 if the checksums match, + * -DER_CSUM if they differ, or another negative rc on a hard I/O/system error. + * + * On a -DER_CSUM mismatch, *got_csum is set to an independent snapshot of the recomputed + * checksum (to be freed by the caller with D_FREE()); left NULL on a match or a hard error. + */ +static int +verify_segment_csum(daos_handle_t coh, daos_key_t *dkey, daos_unit_oid_t *oid, daos_iod_t *iod, + daos_epoch_t epoch, struct dcs_csum_info *ci, struct dcs_csum_info **got_csum) +{ + struct daos_csummer *csummer = NULL; + struct dcs_iod_csums *got_iod_csums = NULL; + d_sg_list_t sgl = {0}; + int rc; + + *got_csum = NULL; + + rc = d_sgl_init(&sgl, 1); + if (!SUCCESS(rc)) + goto out; + + rc = fetch_check_value(coh, oid, dkey, iod, epoch, &sgl); + if (!SUCCESS(rc)) + goto out_sgl; + + rc = daos_csummer_init_with_type(&csummer, ci->cs_type, ci->cs_chunksize, 0); + if (!SUCCESS(rc)) { + D_ERROR("daos_csummer_init_with_type failed: " DF_RC "\n", DP_RC(rc)); + goto out_sgl; + } + /* csummer is local to this call and destroyed below, so no save/restore is needed. + * Only dcs_skip_key_calc is set: it's the only flag daos_csummer_calc_iods() (below) + * consults -- dcs_skip_key_verify/dcs_skip_data_verify belong to other, unused-here + * helpers. */ + csummer->dcs_skip_key_calc = true; + + rc = daos_csummer_calc_iods(csummer, &sgl, iod, NULL, 1, false, NULL, 0, &got_iod_csums); + if (!SUCCESS(rc)) { + D_ERROR("daos_csummer_calc_iods failed: " DF_RC "\n", DP_RC(rc)); + goto out_csummer; + } + + if (!daos_csummer_compare_csum_info(csummer, &got_iod_csums->ic_data[0], ci)) { + rc = snapshot_csum_info(&got_iod_csums->ic_data[0], got_csum); + if (SUCCESS(rc)) + rc = -DER_CSUM; + } + + daos_csummer_free_ic(csummer, &got_iod_csums); +out_csummer: + daos_csummer_destroy(&csummer); +out_sgl: + d_sgl_fini(&sgl, true); +out: + return rc; +} + +static int +check_csum_sv(daos_handle_t coh, daos_key_t *dkey, daos_unit_oid_t *oid, daos_iod_t *iod, + daos_epoch_t epoch, dv_check_csum_cb check_cb, void *cb_arg) +{ + daos_handle_t ioh; + struct dcs_ci_list *cil; + struct dcs_csum_info *ci; + struct dcs_csum_info *got_csum = NULL; + daos_epoch_t sv_epoch; + int rc; + + rc = vos_fetch_begin(coh, *oid, epoch, dkey, 1, iod, VOS_OF_FETCH_CSUM, NULL, &ioh, NULL); + if (!SUCCESS(rc)) { + D_ERROR("vos_fetch_begin for csum check of " DF_UOID " failed: " DF_RC "\n", + DP_UOID(*oid), DP_RC(rc)); + goto out; + } + + cil = vos_ioh2ci(ioh); + sv_epoch = vos_ioh2sv_epoch(ioh); + + if (cil->dcl_csum_infos_nr == 0) { + rc = check_cb(cb_arg, NULL, sv_epoch, cil, NULL); + goto out_fetch_end; + } + + D_ASSERT(cil->dcl_csum_infos_nr == 1); + ci = dcs_csum_info_get(cil, 0); + D_ASSERT(ci_is_valid(ci)); + + rc = verify_segment_csum(coh, dkey, oid, iod, sv_epoch, ci, &got_csum); + if (rc == -DER_CSUM) { + D_DEBUG(DB_IO, "Checksum mismatch of " DF_UOID "\n", DP_UOID(*oid)); + rc = 0; + } + if (!SUCCESS(rc)) { + D_ERROR("Checksum verification of " DF_UOID " failed: " DF_RC "\n", DP_UOID(*oid), + DP_RC(rc)); + goto out_fetch_end; + } + + rc = check_cb(cb_arg, NULL, sv_epoch, cil, &got_csum); + if (!SUCCESS(rc)) + D_DEBUG(DB_IO, "Csum check callback for " DF_UOID " returned: " DF_RC "\n", + DP_UOID(*oid), DP_RC(rc)); + +out_fetch_end: + rc = vos_fetch_end(ioh, NULL, rc); +out: + /* got_csum != NULL is the mismatch signal; check before D_FREE() clears the pointer. */ + if (SUCCESS(rc) && got_csum != NULL) + rc = -DER_CSUM; + D_FREE(got_csum); + return rc; +} + +static int +check_csum_recx(daos_handle_t coh, daos_key_t *dkey, daos_unit_oid_t *oid, daos_iod_t *iod, + daos_epoch_t epoch, dv_check_csum_cb check_cb, void *cb_arg) +{ + daos_handle_t ioh; + struct dcs_ci_list *cil; + struct daos_recx_ep_list *rel; + struct dcs_csum_info **got_csums = NULL; + bool csum_error = false; + int i; + int rc; + + rc = vos_fetch_begin(coh, *oid, epoch, dkey, 1, iod, VOS_OF_FETCH_CSUM, NULL, &ioh, NULL); + if (!SUCCESS(rc)) { + D_ERROR("vos_fetch_begin for csum check of " DF_UOID " failed: " DF_RC "\n", + DP_UOID(*oid), DP_RC(rc)); + goto out; + } + + cil = vos_ioh2ci(ioh); + rel = vos_ioh2recx_list(ioh); + D_ASSERT(rel != NULL); + if (cil->dcl_csum_infos_nr == 0) { + /* no checksum stored: got_csums is left NULL, never allocated below */ + rc = check_cb(cb_arg, rel, 0, cil, got_csums); + goto out_rel; + } + D_ASSERT(cil->dcl_csum_infos_nr == rel->re_nr); + + D_ALLOC_ARRAY(got_csums, cil->dcl_csum_infos_nr); + if (got_csums == NULL) { + rc = -DER_NOMEM; + goto out_rel; + } + + for (i = 0; i < cil->dcl_csum_infos_nr; i++) { + struct dcs_csum_info *ci; + struct daos_recx_ep *rep; + daos_iod_t seg_iod; + + ci = dcs_csum_info_get(cil, i); + D_ASSERT(ci_is_valid(ci)); + rep = &rel->re_items[i]; + + seg_iod = *iod; + seg_iod.iod_recxs = &rep->re_recx; + + rc = verify_segment_csum(coh, dkey, oid, &seg_iod, rep->re_ep, ci, &got_csums[i]); + if (rc == -DER_CSUM) { + D_DEBUG(DB_IO, "Checksum mismatch of " DF_UOID " " DF_RECX "\n", + DP_UOID(*oid), DP_RECX(rep->re_recx)); + csum_error = true; + rc = 0; /* continue checking other segments */ + continue; + } + if (!SUCCESS(rc)) { + D_ERROR("Checksum verification of " DF_UOID " " DF_RECX " failed: " DF_RC + "\n", + DP_UOID(*oid), DP_RECX(rep->re_recx), DP_RC(rc)); + goto out_got_csums; + } + } + + rc = check_cb(cb_arg, rel, 0, cil, got_csums); + if (!SUCCESS(rc)) + D_DEBUG(DB_IO, "Csum check callback for " DF_UOID " returned: " DF_RC "\n", + DP_UOID(*oid), DP_RC(rc)); + +out_got_csums: + for (i = 0; i < cil->dcl_csum_infos_nr; i++) + D_FREE(got_csums[i]); + D_FREE(got_csums); +out_rel: + /* rel ownership is transferred by vos_ioh2recx_list(); free before vos_fetch_end. */ + daos_recx_ep_list_free(rel, iod->iod_nr); + rc = vos_fetch_end(ioh, NULL, rc); +out: + if (rc == 0 && csum_error) + rc = -DER_CSUM; + return rc; +} + +int +dv_check_csum(daos_handle_t poh, struct dv_tree_path *path, daos_epoch_t epoch, + dv_check_csum_cb check_cb, void *cb_arg) +{ + daos_handle_t coh; + daos_iod_t iod = {0}; + int rc = 0; + + /* No-op when no callback is provided; the caller controls whether to consume results. */ + if (check_cb == NULL) + goto out; + + rc = vos_cont_open(poh, path->vtp_cont, &coh); + if (!SUCCESS(rc)) { + D_ERROR("Opening container for csum check of " DF_UOID " failed: " DF_RC "\n", + DP_UOID(path->vtp_oid), DP_RC(rc)); + goto out; + } + + iod.iod_name = path->vtp_akey; + iod.iod_recxs = &path->vtp_recx; + iod.iod_nr = 1; + iod.iod_size = 0; + if (path->vtp_is_recx) { + iod.iod_type = DAOS_IOD_ARRAY; + rc = check_csum_recx(coh, &path->vtp_dkey, &path->vtp_oid, &iod, epoch, check_cb, + cb_arg); + } else { + iod.iod_type = DAOS_IOD_SINGLE; + rc = check_csum_sv(coh, &path->vtp_dkey, &path->vtp_oid, &iod, epoch, check_cb, + cb_arg); + } + + vos_cont_close(coh); +out: + return rc; +} + static void ilog_entry_status(enum ilog_status status, char *status_str, uint32_t status_str_len) { diff --git a/src/utils/ddb/ddb_vos.h b/src/utils/ddb/ddb_vos.h index a1cf34ab957..624f1f71eb7 100644 --- a/src/utils/ddb/ddb_vos.h +++ b/src/utils/ddb/ddb_vos.h @@ -189,6 +189,52 @@ int dv_dump_csum(daos_handle_t poh, struct dv_tree_path *path, daos_epoch_t epoch, dv_dump_csum_cb dump_cb, void *cb_arg); +/** + * Callback invoked by dv_check_csum() with the fetched checksum information and, for each + * checksum entry, whether the checksum recomputed from the currently stored data matches. + * + * @param cb_arg User-provided argument passed through from dv_check_csum(). + * @param recx_rel Recx/epoch list describing the stored extents. Non-NULL for array akeys; + * NULL for single-value akeys. The caller must not free this pointer. + * @param sv_epoch Actual stored epoch of the single value. Non-zero for single-value akeys + * when an SV was found within the requested epoch range; 0 for array akeys + * or when no SV was found (hole or -DER_NONEXIST). + * @param cil Checksum info list. Valid only for the duration of the callback. + * @param got_csums Array of cil->dcl_csum_infos_nr struct dcs_csum_info pointers. got_csums[i] + * holds the checksum recomputed from the currently stored data when entry i + * failed verification against the stored data, and is NULL when entry i + * matched -- got_csums[i] != NULL therefore also serves as the mismatch flag + * for entry i. NULL (the whole array) when cil has 0 entries (nothing to + * verify). Valid only for the duration of the callback; the caller must not + * free it. + * @return 0 on success; a negative error code is propagated back to the caller of + * dv_check_csum(). + */ +typedef int (*dv_check_csum_cb)(void *cb_arg, struct daos_recx_ep_list *recx_rel, + daos_epoch_t sv_epoch, struct dcs_ci_list *cil, + struct dcs_csum_info **got_csums); + +/** + * Fetch the checksum information for the akey identified by \a path, recompute the checksum(s) + * from the currently stored data, and compare them against what is stored on disk. + * + * @param poh Open pool handle. + * @param path VOS tree path identifying the container, object, dkey, and akey. + * For array akeys, path->vtp_recx selects the extent to inspect. + * @param epoch Epoch for the fetch. For single-value akeys, controls which version is + * checked — pass DAOS_EPOCH_MAX to check the latest, or a snapshot epoch to + * check an earlier version. For array akeys, selects the visible extent set. + * @param check_cb Callback invoked with the result. If NULL, the function returns 0 without + * opening the container or calling VOS. + * @param cb_arg Opaque argument forwarded to \a check_cb. + * @return 0 on success (no checksum found, or all checksum(s) matched); -DER_CSUM if + * at least one checksum entry did not match the stored data; another negative + * error code on I/O or system errors. + */ +int +dv_check_csum(daos_handle_t poh, struct dv_tree_path *path, daos_epoch_t epoch, + dv_check_csum_cb check_cb, void *cb_arg); + struct ddb_ilog_entry { uint32_t die_idx; int32_t die_status; diff --git a/src/utils/ddb/tests/ddb_test_driver.c b/src/utils/ddb/tests/ddb_test_driver.c index 61ab707f306..d242613a666 100644 --- a/src/utils/ddb/tests/ddb_test_driver.c +++ b/src/utils/ddb/tests/ddb_test_driver.c @@ -734,7 +734,7 @@ csum_test_recx_setup(struct dt_vos_pool_ctx *tctx, daos_handle_t coh, d_sg_list_ epoch = 1; filler = 'c'; for (recx_idx = 0; recx_idx < DVT_FAKE_RECX_COUNT; recx_idx++) { - recx.rx_idx = recx_idx * csum_ctx->dct_recx_size / 2; + recx.rx_idx = recx_idx * csum_ctx->dct_recx_size / DVT_FAKE_RECX_COUNT; iod.iod_recxs = &recx; memset(buf, filler, csum_ctx->dct_recx_size); @@ -751,7 +751,7 @@ csum_test_recx_setup(struct dt_vos_pool_ctx *tctx, daos_handle_t coh, d_sg_list_ /* g_oids[1]: recxs written with checksum */ epoch = 1; for (recx_idx = 0; recx_idx < DVT_FAKE_RECX_COUNT; recx_idx++) { - recx.rx_idx = recx_idx * csum_ctx->dct_recx_size / 2; + recx.rx_idx = recx_idx * csum_ctx->dct_recx_size / DVT_FAKE_RECX_COUNT; iod.iod_recxs = &recx; memset(buf, filler, csum_ctx->dct_recx_size); @@ -785,6 +785,132 @@ csum_test_recx_setup(struct dt_vos_pool_ctx *tctx, daos_handle_t coh, d_sg_list_ return rc; } +/* + * Write one checksummed value/segment, optionally corrupting its stored checksum (flips + * the first byte) before writing it to VOS. Shared by csum_test_corrupt_sv_setup() (called + * once) and csum_test_corrupt_recx_setup() (called once per overlapping segment). + */ +static int +write_maybe_corrupt(struct daos_csummer *csummer, daos_handle_t coh, daos_unit_oid_t oid, + daos_epoch_t epoch, daos_iod_t *iod, d_sg_list_t *sgl, bool corrupt, + struct dcs_iod_csums **ic_out) +{ + uint8_t *csum_buf; + int rc; + + rc = daos_csummer_calc_iods(csummer, sgl, iod, NULL, 1, false, NULL, 0, ic_out); + if (rc != 0) + return rc; + + if (corrupt) { + csum_buf = ci_idx2csum((*ic_out)->ic_data, 0); + csum_buf[0] ^= 0xff; + } + + return vos_obj_update(coh, oid, epoch, 0, 0, &g_dkeys[0], 1, iod, *ic_out, sgl); +} + +/* + * g_oids[2]: single value written with valid data but a deliberately corrupted (flipped) + * stored checksum, to exercise csum_check's corruption-detection path. + */ +static int +csum_test_corrupt_sv_setup(struct dt_vos_pool_ctx *tctx, daos_handle_t coh, d_sg_list_t *sgl) +{ + struct dt_csum_ctx *csum_ctx; + daos_iod_t iod; + char *buf; + int rc; + + csum_ctx = tctx->dvt_extra; + + D_ALLOC(buf, csum_ctx->dct_sv_size); + if (buf == NULL) + return -DER_NOMEM; + + d_iov_set(&iod.iod_name, g_akeys_str[0], strlen(g_akeys_str[0])); + iod.iod_nr = 1; + iod.iod_type = DAOS_IOD_SINGLE; + iod.iod_size = csum_ctx->dct_sv_size; + iod.iod_recxs = NULL; + + memset(buf, 'd', csum_ctx->dct_sv_size); + d_iov_set(sgl->sg_iovs, &buf[0], csum_ctx->dct_sv_size); + + rc = write_maybe_corrupt(csum_ctx->dct_csummer, coh, g_oids[2], 1, &iod, sgl, true, + &csum_ctx->dct_sv_ic_bad); + if (rc != 0) + daos_csummer_free_ic(csum_ctx->dct_csummer, &csum_ctx->dct_sv_ic_bad); + + D_FREE(buf); + return rc; +} + +/* + * g_oids[2]: recx written with valid data, DVT_FAKE_RECX_COUNT overlapping segments (same + * layout as csum_test_recx_setup()'s g_oids[1] fixture), but only DVT_FAKE_RECX_BAD_IDX's + * stored checksum is deliberately corrupted -- the other segment(s) keep a genuinely + * matching checksum, so csum_check's per-entry mismatch isolation can be exercised (only + * the bad entry should ever be flagged, never the good one(s)). + */ +static int +csum_test_corrupt_recx_setup(struct dt_vos_pool_ctx *tctx, daos_handle_t coh, d_sg_list_t *sgl) +{ + struct dt_csum_ctx *csum_ctx; + daos_iod_t iod; + char *buf; + char filler; + daos_recx_t recx; + int recx_idx; + daos_epoch_t epoch; + int rc; + + csum_ctx = tctx->dvt_extra; + + D_ALLOC(buf, csum_ctx->dct_recx_size); + if (buf == NULL) { + rc = -DER_NOMEM; + goto out; + } + + d_iov_set(&iod.iod_name, g_akeys_str[1], strlen(g_akeys_str[1])); + iod.iod_nr = 1; + iod.iod_type = DAOS_IOD_ARRAY; + iod.iod_size = 1; + recx.rx_nr = csum_ctx->dct_recx_size; + + epoch = 1; + filler = 'f'; + for (recx_idx = 0; recx_idx < DVT_FAKE_RECX_COUNT; recx_idx++) { + recx.rx_idx = recx_idx * csum_ctx->dct_recx_size / DVT_FAKE_RECX_COUNT; + iod.iod_recxs = &recx; + + memset(buf, filler, csum_ctx->dct_recx_size); + d_iov_set(sgl->sg_iovs, &buf[0], csum_ctx->dct_recx_size); + + rc = write_maybe_corrupt(csum_ctx->dct_csummer, coh, g_oids[2], epoch, &iod, sgl, + recx_idx == DVT_FAKE_RECX_BAD_IDX, + &csum_ctx->dct_recx_ics_bad[recx_idx]); + if (rc != 0) + goto out_ics; + + epoch++; + filler++; + } + goto out_buf; + +out_ics: + for (recx_idx = 0; recx_idx < DVT_FAKE_RECX_COUNT; recx_idx++) { + if (csum_ctx->dct_recx_ics_bad[recx_idx] == NULL) + continue; + daos_csummer_free_ic(csum_ctx->dct_csummer, &csum_ctx->dct_recx_ics_bad[recx_idx]); + } +out_buf: + D_FREE(buf); +out: + return rc; +} + /* * Populate the VOS pool with test data for checksum tests. * @@ -814,6 +940,8 @@ ddb_test_csum_setup(void **state) csum_ctx->dct_csum_type = DVT_FAKE_CSUM_TYPE; memset(&csum_ctx->dct_sv_ics[0], 0, sizeof(csum_ctx->dct_sv_ics)); memset(&csum_ctx->dct_recx_ics[0], 0, sizeof(csum_ctx->dct_recx_ics)); + csum_ctx->dct_sv_ic_bad = NULL; + memset(&csum_ctx->dct_recx_ics_bad[0], 0, sizeof(csum_ctx->dct_recx_ics_bad)); rc = daos_csummer_init_with_type(&csum_ctx->dct_csummer, csum_ctx->dct_csum_type, csum_ctx->dct_chunk_size, 0); if (rc != 0) @@ -838,6 +966,12 @@ ddb_test_csum_setup(void **state) if (rc != 0) goto out_sgl; rc = csum_test_recx_setup(tctx, coh, &sgl); + if (rc != 0) + goto out_sgl; + rc = csum_test_corrupt_sv_setup(tctx, coh, &sgl); + if (rc != 0) + goto out_sgl; + rc = csum_test_corrupt_recx_setup(tctx, coh, &sgl); out_sgl: d_sgl_fini(&sgl, false); @@ -880,6 +1014,9 @@ ddb_test_csum_teardown(void **state) daos_csummer_free_ic(csum_ctx->dct_csummer, &csum_ctx->dct_sv_ics[ci_idx]); for (ci_idx = 0; ci_idx < DVT_FAKE_RECX_COUNT; ci_idx++) daos_csummer_free_ic(csum_ctx->dct_csummer, &csum_ctx->dct_recx_ics[ci_idx]); + daos_csummer_free_ic(csum_ctx->dct_csummer, &csum_ctx->dct_sv_ic_bad); + for (ci_idx = 0; ci_idx < DVT_FAKE_RECX_COUNT; ci_idx++) + daos_csummer_free_ic(csum_ctx->dct_csummer, &csum_ctx->dct_recx_ics_bad[ci_idx]); daos_csummer_destroy(&csum_ctx->dct_csummer); vos_cont_destroy(tctx->dvt_poh, csum_ctx->dct_cont_uuid); diff --git a/src/utils/ddb/tests/ddb_test_driver.h b/src/utils/ddb/tests/ddb_test_driver.h index 13457c9a9f1..d0d3247a360 100644 --- a/src/utils/ddb/tests/ddb_test_driver.h +++ b/src/utils/ddb/tests/ddb_test_driver.h @@ -51,7 +51,8 @@ struct dt_vos_pool_ctx { #define DVT_FAKE_SV_SIZE (1u << 10) #define DVT_FAKE_RECX_SIZE (1u << 13) #define DVT_FAKE_CHUNK_SIZE (1u << 12) -#define DVT_FAKE_CSUM_TYPE (HASH_TYPE_CRC64) +#define DVT_FAKE_CSUM_TYPE (HASH_TYPE_CRC64) +#define DVT_FAKE_RECX_BAD_IDX (1) struct dt_csum_ctx { uuid_t dct_cont_uuid; @@ -62,6 +63,8 @@ struct dt_csum_ctx { struct daos_csummer *dct_csummer; struct dcs_iod_csums *dct_sv_ics[DVT_FAKE_SV_COUNT]; struct dcs_iod_csums *dct_recx_ics[DVT_FAKE_RECX_COUNT]; + struct dcs_iod_csums *dct_sv_ic_bad; + struct dcs_iod_csums *dct_recx_ics_bad[DVT_FAKE_RECX_COUNT]; }; daos_unit_oid_t dvt_gen_uoid(uint32_t i); diff --git a/src/utils/ddb/tests/ddb_vos_tests.c b/src/utils/ddb/tests/ddb_vos_tests.c index 484876b1fa7..da625f47e3c 100644 --- a/src/utils/ddb/tests/ddb_vos_tests.c +++ b/src/utils/ddb/tests/ddb_vos_tests.c @@ -1478,6 +1478,283 @@ dump_csum_recx_tests(void **state) assert_rc_equal(-DER_INVAL, rc); } +/* Callback that returns *(int *)cb_args, or 0 if cb_args is NULL. */ +static int +check_cb_return_rc(void *cb_args, struct daos_recx_ep_list *recx_rel, daos_epoch_t sv_epoch, + struct dcs_ci_list *cil, struct dcs_csum_info **got_csums) +{ + return (cb_args != NULL) ? (*(int *)cb_args) : (0); +} + +static void +check_csum_error_tests(void **state) +{ + struct dt_vos_pool_ctx *tctx = *state; + struct dt_csum_ctx *csum_ctx = tctx->dvt_extra; + struct dv_tree_path path = {0}; + int rc; + + uuid_copy(path.vtp_cont, csum_ctx->dct_cont_uuid); + path.vtp_dkey = g_dkeys[0]; + path.vtp_akey = g_akeys[0]; /* single value type */ + path.vtp_is_recx = false; + + /* invalid poh: error comes from vos_cont_open */ + rc = dv_check_csum(DAOS_HDL_INVAL, &path, DAOS_EPOCH_MAX, check_cb_return_rc, NULL); + assert_rc_equal(-DER_INVAL, rc); +} + +/* + * g_oids[0]: SV stored at epoch 1 without checksum. + * Fetching at EPOCH_MAX finds the SV (sv_epoch=1) but cil is empty (no checksum stored), so + * there is nothing to verify and got_csums is NULL. + */ +static int +verify_csum_sv_cb_001(void *cb_args, struct daos_recx_ep_list *recx_rel, daos_epoch_t sv_epoch, + struct dcs_ci_list *cil, struct dcs_csum_info **got_csums) +{ + assert_null(cb_args); + assert_null(recx_rel); + assert_int_equal(sv_epoch, 1); + assert_non_null(cil); + assert_int_equal(cil->dcl_csum_infos_nr, 0); + assert_null(got_csums); + + return 0; +} + +/* g_oids[1]: SV stored at epoch 1 with a valid, matching checksum. */ +static int +verify_csum_sv_cb_002(void *cb_args, struct daos_recx_ep_list *recx_rel, daos_epoch_t sv_epoch, + struct dcs_ci_list *cil, struct dcs_csum_info **got_csums) +{ + assert_non_null(cb_args); + assert_null(recx_rel); + assert_int_equal(sv_epoch, 1); + assert_non_null(cil); + assert_int_equal(cil->dcl_csum_infos_nr, 1); + assert_non_null(got_csums); + assert_null(got_csums[0]); + + return 0; +} + +/* g_oids[1]: SV stored at epoch 2 (latest) with a valid, matching checksum. */ +static int +verify_csum_sv_cb_002_latest(void *cb_args, struct daos_recx_ep_list *recx_rel, + daos_epoch_t sv_epoch, struct dcs_ci_list *cil, + struct dcs_csum_info **got_csums) +{ + assert_non_null(cb_args); + assert_null(recx_rel); + assert_int_equal(sv_epoch, 2); + assert_non_null(cil); + assert_int_equal(cil->dcl_csum_infos_nr, 1); + assert_non_null(got_csums); + assert_null(got_csums[0]); + + return 0; +} + +/* + * g_oids[2]: SV stored with valid data but a deliberately corrupted stored checksum (fixture + * flips the first byte of an otherwise-correct checksum before writing it -- see + * csum_test_corrupt_sv_setup()). got_csums[0] should hold the checksum *recomputed* from the + * still-valid data, i.e. the stored value with that one byte's flip undone. + */ +static int +verify_csum_sv_cb_003(void *cb_args, struct daos_recx_ep_list *recx_rel, daos_epoch_t sv_epoch, + struct dcs_ci_list *cil, struct dcs_csum_info **got_csums) +{ + struct dcs_csum_info *stored_ci; + struct dcs_csum_info *got_ci; + uint8_t expect_byte0; + + assert_non_null(cb_args); + assert_null(recx_rel); + assert_int_equal(sv_epoch, 1); + assert_non_null(cil); + assert_int_equal(cil->dcl_csum_infos_nr, 1); + + assert_non_null(got_csums); + assert_non_null(got_csums[0]); + stored_ci = dcs_csum_info_get(cil, 0); + got_ci = got_csums[0]; + expect_byte0 = ci_idx2csum(stored_ci, 0)[0] ^ 0xff; + assert_int_equal(got_ci->cs_len, stored_ci->cs_len); + assert_int_equal(got_ci->cs_nr, stored_ci->cs_nr); + assert_int_equal(ci_idx2csum(got_ci, 0)[0], expect_byte0); + assert_memory_equal(ci_idx2csum(got_ci, 0) + 1, ci_idx2csum(stored_ci, 0) + 1, + stored_ci->cs_len - 1); + + return 0; +} + +static void +check_csum_sv_tests(void **state) +{ + struct dt_vos_pool_ctx *tctx = *state; + struct dt_csum_ctx *csum_ctx = tctx->dvt_extra; + struct dv_tree_path path = {0}; + int rc; + + uuid_copy(path.vtp_cont, csum_ctx->dct_cont_uuid); + path.vtp_dkey = g_dkeys[0]; + path.vtp_akey = g_akeys[0]; /* single value type */ + path.vtp_is_recx = false; + + /* no csum info: nothing to verify, success */ + path.vtp_oid = g_oids[0]; + rc = dv_check_csum(tctx->dvt_poh, &path, DAOS_EPOCH_MAX, verify_csum_sv_cb_001, NULL); + assert_success(rc); + + /* valid, matching csum info: epoch 1 returns the epoch-1 checksum */ + path.vtp_oid = g_oids[1]; + rc = dv_check_csum(tctx->dvt_poh, &path, 1, verify_csum_sv_cb_002, csum_ctx); + assert_success(rc); + + /* valid, matching csum info: EPOCH_MAX returns the latest (epoch-2) checksum */ + path.vtp_oid = g_oids[1]; + rc = dv_check_csum(tctx->dvt_poh, &path, DAOS_EPOCH_MAX, verify_csum_sv_cb_002_latest, + csum_ctx); + assert_success(rc); + + /* deliberately corrupted csum info: -DER_CSUM */ + path.vtp_oid = g_oids[2]; + rc = dv_check_csum(tctx->dvt_poh, &path, DAOS_EPOCH_MAX, verify_csum_sv_cb_003, csum_ctx); + assert_rc_equal(-DER_CSUM, rc); + + /* with csum info, without callback */ + path.vtp_oid = g_oids[1]; + rc = dv_check_csum(tctx->dvt_poh, &path, DAOS_EPOCH_MAX, NULL, csum_ctx); + assert_success(rc); + + /* callback failure is propagated */ + path.vtp_oid = g_oids[1]; + rc = dv_check_csum(tctx->dvt_poh, &path, DAOS_EPOCH_MAX, check_cb_return_rc, + &(int){-DER_INVAL}); + assert_rc_equal(-DER_INVAL, rc); +} + +/* g_oids[0]: recxs stored without checksum. */ +static int +verify_csum_recx_cb_001(void *cb_args, struct daos_recx_ep_list *recx_rel, daos_epoch_t sv_epoch, + struct dcs_ci_list *cil, struct dcs_csum_info **got_csums) +{ + assert_null(cb_args); + assert_non_null(recx_rel); + assert_int_equal(sv_epoch, 0); + assert_non_null(cil); + assert_int_equal(cil->dcl_csum_infos_nr, 0); + assert_null(got_csums); + + return 0; +} + +/* g_oids[1]: recxs stored with valid, matching checksums (2 overlapping segments). */ +static int +verify_csum_recx_cb_002(void *cb_args, struct daos_recx_ep_list *recx_rel, daos_epoch_t sv_epoch, + struct dcs_ci_list *cil, struct dcs_csum_info **got_csums) +{ + assert_non_null(cb_args); + assert_non_null(recx_rel); + assert_int_equal(sv_epoch, 0); + assert_non_null(cil); + assert_int_equal(cil->dcl_csum_infos_nr, DVT_FAKE_RECX_COUNT); + assert_non_null(got_csums); + assert_null(got_csums[0]); + assert_null(got_csums[1]); + + return 0; +} + +/* + * g_oids[2]: recx stored with valid data, DVT_FAKE_RECX_COUNT overlapping segments (same + * layout as verify_csum_recx_cb_002()/g_oids[1]), but only DVT_FAKE_RECX_BAD_IDX's stored + * checksum is deliberately corrupted (single-byte flip, same fixture style as + * verify_csum_sv_cb_003() above) -- the other segment(s) keep a genuinely matching + * checksum. Exercises that got_csums[] correctly isolates the corrupted entry without + * affecting the others (the exact scenario behind the flat-buffer indexing bug fixed in + * 65dacd2e4a). + */ +static int +verify_csum_recx_cb_003(void *cb_args, struct daos_recx_ep_list *recx_rel, daos_epoch_t sv_epoch, + struct dcs_ci_list *cil, struct dcs_csum_info **got_csums) +{ + struct dcs_csum_info *stored_ci; + struct dcs_csum_info *got_ci; + uint8_t expect_byte0; + int i; + + assert_non_null(cb_args); + assert_non_null(recx_rel); + assert_int_equal(sv_epoch, 0); + assert_non_null(cil); + assert_int_equal(cil->dcl_csum_infos_nr, DVT_FAKE_RECX_COUNT); + assert_non_null(got_csums); + + for (i = 0; i < DVT_FAKE_RECX_COUNT; i++) { + if (i != DVT_FAKE_RECX_BAD_IDX) { + assert_null(got_csums[i]); + continue; + } + + assert_non_null(got_csums[i]); + stored_ci = dcs_csum_info_get(cil, i); + got_ci = got_csums[i]; + expect_byte0 = ci_idx2csum(stored_ci, 0)[0] ^ 0xff; + assert_int_equal(got_ci->cs_len, stored_ci->cs_len); + assert_int_equal(got_ci->cs_nr, stored_ci->cs_nr); + assert_int_equal(ci_idx2csum(got_ci, 0)[0], expect_byte0); + assert_memory_equal(ci_idx2csum(got_ci, 0) + 1, ci_idx2csum(stored_ci, 0) + 1, + stored_ci->cs_len - 1); + } + + return 0; +} + +static void +check_csum_recx_tests(void **state) +{ + struct dt_vos_pool_ctx *tctx = *state; + struct dt_csum_ctx *csum_ctx = tctx->dvt_extra; + struct dv_tree_path path = {0}; + int rc; + + uuid_copy(path.vtp_cont, csum_ctx->dct_cont_uuid); + path.vtp_dkey = g_dkeys[0]; + path.vtp_akey = g_akeys[1]; /* array value type */ + path.vtp_is_recx = true; + path.vtp_recx.rx_idx = 0; + path.vtp_recx.rx_nr = csum_ctx->dct_recx_size; + + /* no csum info */ + path.vtp_oid = g_oids[0]; + rc = dv_check_csum(tctx->dvt_poh, &path, DAOS_EPOCH_MAX, verify_csum_recx_cb_001, NULL); + assert_success(rc); + + /* valid, matching csum info */ + path.vtp_oid = g_oids[1]; + rc = dv_check_csum(tctx->dvt_poh, &path, DAOS_EPOCH_MAX, verify_csum_recx_cb_002, csum_ctx); + assert_success(rc); + + /* deliberately corrupted csum info: -DER_CSUM */ + path.vtp_oid = g_oids[2]; + rc = dv_check_csum(tctx->dvt_poh, &path, DAOS_EPOCH_MAX, verify_csum_recx_cb_003, csum_ctx); + assert_rc_equal(-DER_CSUM, rc); + + /* with csum info, without callback */ + path.vtp_oid = g_oids[1]; + rc = dv_check_csum(tctx->dvt_poh, &path, DAOS_EPOCH_MAX, NULL, csum_ctx); + assert_success(rc); + + /* callback failure is propagated */ + path.vtp_oid = g_oids[1]; + rc = dv_check_csum(tctx->dvt_poh, &path, DAOS_EPOCH_MAX, check_cb_return_rc, + &(int){-DER_INVAL}); + assert_rc_equal(-DER_INVAL, rc); +} + /* * All these tests use the same VOS tree that is created at suit_setup. Therefore, tests * that modify the state of the tree (delete, add, etc) should be run after all others. @@ -1516,6 +1793,9 @@ const struct CMUnitTest dv_test_cases[] = { TEST_CSUM(dump_csum_error_tests), TEST_CSUM(dump_csum_sv_tests), TEST_CSUM(dump_csum_recx_tests), + TEST_CSUM(check_csum_error_tests), + TEST_CSUM(check_csum_sv_tests), + TEST_CSUM(check_csum_recx_tests), }; int