diff --git a/src/main/java/com/yunhai/formcenter/controller/api/FormAnswerReadController.java b/src/main/java/com/yunhai/formcenter/controller/api/FormAnswerReadController.java index 0f2e1d6..6d299db 100644 --- a/src/main/java/com/yunhai/formcenter/controller/api/FormAnswerReadController.java +++ b/src/main/java/com/yunhai/formcenter/controller/api/FormAnswerReadController.java @@ -1,7 +1,9 @@ package com.yunhai.formcenter.controller.api; +import com.yunhai.formcenter.model.ImportModels; import com.yunhai.formcenter.model.ReadModels; import com.yunhai.formcenter.service.AnswerExportService; +import com.yunhai.formcenter.service.AnswerImportService; import com.yunhai.formcenter.service.AnswerReadService; import com.yunhai.formcenter.service.AnswerStatsService; import com.yunhai.formcenter.service.DataScopeService; @@ -18,6 +20,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletResponse; import java.io.IOException; @@ -55,6 +58,7 @@ public class FormAnswerReadController { private final AnswerReadService readService; private final AnswerStatsService statsService; private final AnswerExportService exportService; + private final AnswerImportService importService; /** 两段式列表(keyset 游标对外是 uuid,排序恒为提交时间序) */ @PostMapping("/list") @@ -122,6 +126,46 @@ public class FormAnswerReadController { } } + /** + * 导入模板下载(契约 v2 C14,T18)。表头=当前发布版本的字段 label,第一列固定"被记录人ID", + * 第二行是给人看的说明;与 {@link #importAnswers} 的解析共用同一份映射(都在 {@code AnswerImportService})。 + * + *

守卫与导出同一把(该表单 ALL 档)——模板本身就是数据面产物(它给出这张表的字段清单), + * 且拿到模板的人下一步就是要写数据。未发布的表单没有可导入的字段定义,服务侧直接拒。

+ */ + @GetMapping("/import-template") + public void importTemplate(BoaUser user, + @RequestParam(required = false) Long schId, + @RequestParam("formUuid") String formUuid, + HttpServletResponse response) throws IOException { + requireFormUuid(formUuid); + Long domain = adminGuard.requireData(user, schId, defService.findFormForGuard(formUuid)); + String fileName = "form-import-template.xlsx"; + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Disposition", + "attachment; filename=" + fileName + "; filename*=UTF-8''" + encode(fileName)); + try (OutputStream out = response.getOutputStream()) { + importService.template(domain, formUuid, out); + } + } + + /** + * 批量导入(C14/C13,T18):multipart xlsx,逐行走提交管线(V60),响应逐行结果不静默坏行。 + * + *

这条端点返回 JSON 信封(与导出不同:它不是文件流)。幂等键由服务端按 C13 确定性生成, + * 所以"重复导入同一个文件"是逐行 200+原 answerId,而不是重复落库。

+ */ + @PostMapping("/import") + public ImportModels.Result importAnswers(BoaUser user, + @RequestParam(required = false) Long schId, + @RequestParam("formUuid") String formUuid, + @RequestParam("file") MultipartFile file) { + requireFormUuid(formUuid); + Long domain = adminGuard.requireData(user, schId, defService.findFormForGuard(formUuid)); + return importService.importAnswers(user, domain, formUuid, file); + } + private static void requireFormUuid(String formUuid) { if (formUuid == null || formUuid.trim().isEmpty()) { throw new BizException(ErrorCode.PARAM_ERROR, "formUuid 不能为空(数据面按表单 uuid 寻址,契约 §1)"); diff --git a/src/main/java/com/yunhai/formcenter/model/ImportModels.java b/src/main/java/com/yunhai/formcenter/model/ImportModels.java new file mode 100644 index 0000000..7af3def --- /dev/null +++ b/src/main/java/com/yunhai/formcenter/model/ImportModels.java @@ -0,0 +1,44 @@ +package com.yunhai.formcenter.model; + +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +/** + * 批量导入的响应模型(契约 v2 C14,T18)。 + * + *

逐行结果,坏行不静默:导入是"把 Excel 里的一堆行交给提交管线",管线对每一行都可能给出 + * 不同的答案(值校验/必填/截止/限次/被记录人不存在…),所以响应形状是每行一条 {@link RowResult}, + * 而不是"整体成功/失败"。整批 500 行里 499 行成功必须能从响应里看出那 1 行为什么失败。

+ */ +public final class ImportModels { + + private ImportModels() { + } + + /** {@code POST /api/form/admin/data/import} 的 data */ + @Getter + @Setter + public static class Result { + /** 逐行结果,按文件里的行号顺序({@code rowNo} 是 xlsx 的物理行号,1 起) */ + private List rows; + private int okCount; + private int failCount; + } + + /** 单行结果(C14 的字段闭集就这三件可选两件:{@code {rowNo, ok, answerId?, error?}},不加扩展位) */ + @Getter + @Setter + public static class RowResult { + private Integer rowNo; + private Boolean ok; + /** + * 成功时的答卷 uuid。命中幂等键(重复导入同一个文件,C13)时是原答卷 uuid—— + * 契约明令"逐行返回 200+原 answerId",所以这种情况同样算 ok,不额外给"重复"标记位。 + */ + private String answerId; + /** 失败原因=管线给的那句话原样(不在导入侧改写,否则同一个校验在两个地方有两种说法) */ + private String error; + } +} diff --git a/src/main/java/com/yunhai/formcenter/service/AnswerImportService.java b/src/main/java/com/yunhai/formcenter/service/AnswerImportService.java new file mode 100644 index 0000000..e0e6aa9 --- /dev/null +++ b/src/main/java/com/yunhai/formcenter/service/AnswerImportService.java @@ -0,0 +1,473 @@ +package com.yunhai.formcenter.service; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yunhai.formcenter.common.FormSchemaValidator; +import com.yunhai.formcenter.entity.Form; +import com.yunhai.formcenter.model.AnswerModels; +import com.yunhai.formcenter.model.ImportModels; +import com.yunhai.platform.apistd.BizException; +import com.yunhai.platform.utils.exception.ErrorCode; +import com.yunhai.platform.utils.model.BoaUser; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 批量导入(契约 v2 §5.1/C13/C14,T18,验收 V60/V74)。 + * + *

导入不是第二条写库路径:每一行都拼成一次普通提交请求进 {@link AnswerService#submit} + * (V60 采集管线唯一),所以值校验、按钉住版本解释、附件归属、限次、截止、幂等、索引、事件 + * 与手工填报逐字同一套——"导入数据与手工填的数据完全可比"(§3.4)靠的就是这个, + * 而不是导入侧再写一份校验。本类只做四件事:读 xlsx、按 label 映射列、算确定性幂等键、 + * 逐行收集结果。

+ * + *

四条纪律:

+ *
    + *
  1. 坏行不静默:一行的失败不影响其它行(逐行独立调用管线并独立捕获),响应里逐行给 + * {@code {rowNo, ok, answerId|error}};失败原因是管线原话,导入侧不改写;
  2. + *
  3. 幂等键确定性(C13):{@code sha256("import:" + 表单 uuid + "|" + subject + "|" + 规范化值文档)}。 + * "批次串"取表单坐标而不是请求级随机 uuid(那等于没有幂等),也不是文件字节摘要—— + * 字节摘要会让"改一个字重传整份"把未改的行全部重插一遍。详见 {@link #idempotencyKey}; + * 落地差额登记在手册 §5 P128;
  4. + *
  5. 表头先校验再读数据(C14):重名 label、当前版本没有的列、一列都不认识,都在碰到任何一行之前拒。 + * "认不出的列就当没有"会让那一列的数据静默消失,比报错糟得多;
  6. + *
  7. 行上限 1000,超限整批拒:截断会让"导入完成"与"文件里有多少行"两个数不一致且无从发现。
  8. + *
+ */ +@Slf4j +@Service +@RequiredArgsConstructor +public class AnswerImportService { + + /** C14:单次导入行上限(超限 PARAM_ERROR) */ + static final int MAX_ROWS = 1000; + + /** 模板第一列的表头(可空;填 UserBase 用户主键=被记录人,即代录语义) */ + static final String SUBJECT_HEADER = "被记录人ID"; + + /** 模板第二行的说明行前缀(读侧据此跳过;用户手动删掉也不影响解析) */ + private static final String NOTE_PREFIX = "说明:"; + + /** 导入文件大小上限:xlsx 是纯数据,与附件上传的 50MB 不同轴,不共用那个常量 */ + private static final long MAX_FILE_BYTES = 10L * 1024 * 1024; + + /** 幂等键长度:列宽 64,取 40 位十六进制留足余量 */ + private static final int IDEMPOTENCY_KEY_LENGTH = 40; + + private static final List MULTI_OPTION_TYPES = Arrays.asList("checkbox", "multiselect"); + private static final Set UNSUPPORTED_TYPES = new HashSet<>(Arrays.asList("upload", "image")); + + private final FormDefService defService; + private final FormDictService dictService; + private final AnswerService answerService; + + // ==================== 模板(C14 的下载件) ==================== + + /** + * 生成导入模板:表头=当前发布版本的字段 label(前面固定一列"被记录人ID"),第二行是给填表人看的说明。 + * + *

列序取当前发布版本的字段序(与导出同源那条规则)——导入的行会钉在这个版本上, + * 模板给旧版本的列会让每一行都撞上"未知字段"。

+ */ + public void template(Long domainSchId, String formUuid, OutputStream out) { + Form form = defService.readDomainForm(domainSchId, formUuid); + Map fields = publishedFields(form); + requireUniqueLabels(fields); + EasyExcel.write(out).head(headOf(fields)).sheet("导入").doWrite(new ArrayList<>()); + } + + // ==================== 导入 ==================== + + /** + * 逐行导入。 + * + * @param domainSchId 已由控制器过 {@code requireData}(该表单 ALL 档)的租户域 + */ + public ImportModels.Result importAnswers(BoaUser user, Long domainSchId, String formUuid, MultipartFile file) { + byte[] bytes = requireFile(file); + Form form = defService.readDomainForm(domainSchId, formUuid); + Map fields = publishedFields(form); + + List rows = readRows(bytes); + if (rows.size() - 1 > MAX_ROWS) { + throw new BizException(ErrorCode.PARAM_ERROR, + "单次导入行数上限 " + MAX_ROWS + " 行(当前 " + (rows.size() - 1) + " 行),请拆分文件"); + } + Map columnByKey = mapColumns(rows.get(0).cells, fields); + + ImportModels.Result result = new ImportModels.Result(); + List items = new ArrayList<>(); + int ok = 0; + for (SheetRow row : rows.subList(1, rows.size())) { + ImportModels.RowResult item = new ImportModels.RowResult(); + item.setRowNo(row.rowNo); + try { + AnswerModels.SubmitView view = answerService.submit(user, + requestOf(form, row, fields, columnByKey)); + item.setOk(true); + item.setAnswerId(view.getAnswerId()); + ok++; + } catch (BizException e) { + item.setOk(false); + item.setError(e.getMessage()); + } catch (RuntimeException e) { + // 单行的意外异常不能带倒整批(与批量软删/批量改状态同一条"逐条独立"的既有口径) + item.setOk(false); + item.setError("导入该行时服务内部异常:" + e.getClass().getSimpleName()); + log.warn("[导入] 第 {} 行异常", row.rowNo, e); + } + items.add(item); + } + result.setRows(items); + result.setOkCount(ok); + result.setFailCount(items.size() - ok); + log.info("[导入] form={}/{} 数据行={} 成功={} 失败={}", + form.getAppCode(), form.getCode(), items.size(), ok, result.getFailCount()); + return result; + } + + // ==================== 行 → 提交请求 ==================== + + private AnswerModels.SubmitRequest requestOf(Form form, SheetRow row, Map fields, + Map columnByKey) { + AnswerModels.SubmitRequest req = new AnswerModels.SubmitRequest(); + req.setFormCode(form.getCode()); + req.setAppCode(form.getAppCode()); + req.setSubjectId(subjectOf(row)); + req.setValues(valuesOf(row, fields, columnByKey)); + req.setIdempotencyKey(idempotencyKey(form.getUuid(), req.getSubjectId(), req.getValues())); + return req; + } + + private static Integer subjectOf(SheetRow row) { + String raw = trim(row.cells.get(0)); + if (raw == null || raw.isEmpty()) { + return null; + } + try { + // Excel 的数字单元格可能带小数尾巴("10133.0"),按整数取 + return Integer.valueOf(new BigDecimal(raw).toBigInteger().toString()); + } catch (NumberFormatException e) { + throw new BizException(ErrorCode.PARAM_ERROR, "被记录人ID 必须是整数:\"" + raw + "\""); + } + } + + private ObjectNode valuesOf(SheetRow row, Map fields, Map columnByKey) { + ObjectNode values = FormSchemaValidator.mapper().createObjectNode(); + for (Map.Entry column : columnByKey.entrySet()) { + String key = column.getValue(); + JsonNode field = fields.get(key); + String raw = trim(row.cells.get(column.getKey())); + if (raw == null || raw.isEmpty()) { + continue; + } + String type = field == null ? "" : field.path("type").asText(""); + if (UNSUPPORTED_TYPES.contains(type)) { + throw new BizException(ErrorCode.PARAM_ERROR, + "字段「" + labelOf(field, key) + "」是附件类(" + type + "),批量导入不支持上传," + + "请逐条填写(" + key + ")"); + } + values.set(key, valueNode(raw, type)); + } + return values; + } + + /** + * 单元格文本 → 值节点。只做最小的类型转换:能转成数字就转、多选取成数组,其余一律留字符串—— + * 格式对不对由提交管线里那一份 {@code FormValueValidator} 判。数字格转不动时也原样给字符串, + * 让校验器说出与手工填报同一句"必须是数字",而不是导入侧编一条。 + */ + private static JsonNode valueNode(String raw, String type) { + if ("number".equals(type)) { + try { + return FormSchemaValidator.mapper().getNodeFactory().numberNode(new BigDecimal(raw)); + } catch (NumberFormatException e) { + return FormSchemaValidator.mapper().getNodeFactory().textNode(raw); + } + } + if (MULTI_OPTION_TYPES.contains(type)) { + ArrayNode array = FormSchemaValidator.mapper().createArrayNode(); + for (String token : raw.split("[、,,]")) { + String value = token.trim(); + if (!value.isEmpty()) { + array.add(value); + } + } + return array; + } + return FormSchemaValidator.mapper().getNodeFactory().textNode(raw); + } + + // ==================== 表头 ==================== + + /** 表头两行:第一行=label(解析按它映射字段),第二行=说明(读侧按前缀跳过) */ + private static List> headOf(Map fields) { + List> head = new ArrayList<>(); + head.add(Arrays.asList(SUBJECT_HEADER, + NOTE_PREFIX + "填 UserBase 用户ID;留空=以你本人为被记录人(代录需该表单 ALL 档)")); + for (Map.Entry entry : fields.entrySet()) { + head.add(Arrays.asList(labelOf(entry.getValue(), entry.getKey()), hintOf(entry.getValue(), entry.getKey()))); + } + return head; + } + + private static String hintOf(JsonNode field, String key) { + StringBuilder hint = new StringBuilder(NOTE_PREFIX); + hint.append(field.path("required").asBoolean(false) ? "必填;" : "选填;"); + String type = field.path("type").asText(""); + if ("number".equals(type)) { + hint.append("数字"); + } else if (MULTI_OPTION_TYPES.contains(type)) { + hint.append("多选,多个值用、分隔"); + } else if ("select".equals(type) || "radio".equals(type)) { + hint.append("候选值之一:").append(optionsOf(field)); + } else if ("date".equals(type)) { + hint.append("日期 yyyy-MM-dd"); + } else if ("datetime".equals(type)) { + hint.append("时间 yyyy-MM-dd HH:mm:ss"); + } else if (UNSUPPORTED_TYPES.contains(type)) { + hint.append("附件类,导入不支持,请留空"); + } else { + hint.append("文本"); + } + return hint.toString(); + } + + private static String optionsOf(JsonNode field) { + JsonNode options = field.path("options"); + if (!options.isArray()) { + return "(见表单定义)"; + } + List values = new ArrayList<>(); + for (JsonNode option : options) { + values.add(option.path("value").asText(option.path("label").asText("?"))); + } + return String.join("、", values); + } + + private static Map countLabels(Map fields) { + Map counts = new LinkedHashMap<>(); + for (Map.Entry entry : fields.entrySet()) { + counts.merge(labelOf(entry.getValue(), entry.getKey()), 1, Integer::sum); + } + return counts; + } + + /** + * label 唯一性(C14"重名 label 拒绝并提示先改 label"):模板与解析都按 label 映射列, + * 重名就等于"这一列到底是哪个字段"没有答案。模板生成时同样先判—— + * 生成一份带歧义的模板再在导入时报错,不如在下载这一步就把设计器的问题点名。 + * + *

另一件保留名:字段不能就叫 {@code 被记录人ID},那一列是模板的固定第一列(代录入口)。

+ */ + private static void requireUniqueLabels(Map fields) { + for (Map.Entry entry : fields.entrySet()) { + String label = labelOf(entry.getValue(), entry.getKey()); + if (SUBJECT_HEADER.equals(label)) { + throw new BizException(ErrorCode.PARAM_ERROR, + "字段 label 不能叫「" + SUBJECT_HEADER + "」(它是模板的固定第一列)"); + } + if (countLabels(fields).get(label) > 1) { + throw new BizException(ErrorCode.PARAM_ERROR, + "字段 label「" + label + "」重复,模板与解析都按 label 映射,请先在设计器改 label 再导入"); + } + } + } + + private static String labelOf(JsonNode field, String key) { + String label = field == null ? null : trim(field.path("label").asText(null)); + return label == null || label.isEmpty() ? key : label; + } + + /** + * 表头 → {@code 列号 → 字段 key}(C14"表头按 label 映射字段")。 + * + *

三道检查都在读数据之前:① 字段 label 有重复(模板与解析都按 label 映射,重复即二义, + * 提示先改 label);② 有列名在当前发布版本里不存在(模板过期或手改表头);③ 一列都不认识。 + * ②若按"忽略那一列"处理,那一列的数据会静默消失。

+ */ + private static Map mapColumns(Map head, Map fields) { + requireUniqueLabels(fields); + Map keyByLabel = new HashMap<>(); + for (Map.Entry entry : fields.entrySet()) { + keyByLabel.put(labelOf(entry.getValue(), entry.getKey()), entry.getKey()); + } + + Map columnByKey = new LinkedHashMap<>(); + List unknown = new ArrayList<>(); + for (Map.Entry cell : head.entrySet()) { + if (cell.getKey() == 0) { + continue; + } + String label = trim(cell.getValue()); + if (label == null || label.isEmpty()) { + continue; + } + String key = keyByLabel.get(label); + if (key == null) { + unknown.add(label); + } else { + columnByKey.put(cell.getKey(), key); + } + } + if (!unknown.isEmpty()) { + throw new BizException(ErrorCode.PARAM_ERROR, + "表头有当前发布版本里不存在的列:" + String.join("、", unknown) + + "(请用本页下载的模板,或先发布含这些字段的版本)"); + } + if (columnByKey.isEmpty()) { + throw new BizException(ErrorCode.PARAM_ERROR, "表头没有任何可识别的字段列(不是这张表单的模板?)"); + } + return columnByKey; + } + + // ==================== 读文件 ==================== + + private static byte[] requireFile(MultipartFile file) { + if (file == null || file.isEmpty()) { + throw new BizException(ErrorCode.PARAM_ERROR, "导入文件为空"); + } + if (file.getSize() > MAX_FILE_BYTES) { + throw new BizException(ErrorCode.PARAM_ERROR, "导入文件过大(≤10MB)"); + } + try { + return file.getBytes(); + } catch (IOException e) { + throw new BizException(ErrorCode.PARAM_ERROR, "导入文件读取失败:" + e.getMessage()); + } + } + + /** + * 读工作表:返回的列表第 0 项是表头行,其后是数据行(模板的说明行被识别后丢弃)。 + * + *

{@code headRowNumber(1)}:EasyExcel 用第一行做表头({@link AnalysisEventListener#invokeHeadMap}), + * 其余按数据回调。行号取 {@code rowIndex + 1}——与用户在 Excel 里看到的行号一致, + * 报错时"第 7 行"要能直接指到那一格。

+ */ + private static List readRows(byte[] bytes) { + List dataRows = new ArrayList<>(); + Map head = new LinkedHashMap<>(); + try (InputStream in = new ByteArrayInputStream(bytes)) { + EasyExcel.read(in, new AnalysisEventListener>() { + @Override + public void invokeHeadMap(Map headMap, AnalysisContext context) { + head.putAll(headMap); + } + + @Override + public void invoke(Map row, AnalysisContext context) { + dataRows.add(new SheetRow(context.readRowHolder().getRowIndex() + 1, + new LinkedHashMap<>(row))); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // 逐行已在 invoke 里收集,无需收尾 + } + }).sheet().headRowNumber(1).doRead(); + } catch (IOException | RuntimeException e) { + throw new BizException(ErrorCode.PARAM_ERROR, + "导入文件解析失败(需为本页下载的 xlsx 模板):" + e.getMessage()); + } + if (head.isEmpty()) { + throw new BizException(ErrorCode.PARAM_ERROR, "导入文件没有表头行"); + } + if (!dataRows.isEmpty() && startsWithNote(dataRows.get(0).cells.get(0))) { + dataRows.remove(0); + } + if (dataRows.isEmpty()) { + throw new BizException(ErrorCode.PARAM_ERROR, "表头之下没有数据行"); + } + List rows = new ArrayList<>(); + rows.add(new SheetRow(1, head)); + rows.addAll(dataRows); + return rows; + } + + private static boolean startsWithNote(String cell) { + String value = trim(cell); + return value != null && value.startsWith(NOTE_PREFIX); + } + + /** 当前发布版本的字段表;未发布直接拒(没有可校验的 schema,也没有可钉的版本) */ + private Map publishedFields(Form form) { + if (form.getCurrentVersion() == null) { + throw new BizException(ErrorCode.PARAM_ERROR, + "表单尚未发布,没有可导入的字段定义:" + form.getAppCode() + "/" + form.getCode()); + } + String schemaJson = defService.pinnedForData(form.getUuid(), form.getCurrentVersion()) + .versionRow().getSchemaJson(); + return FormSchemaValidator.fieldsByKeys(dictService.expandSchema(form.getSchId(), schemaJson)); + } + + // ==================== 幂等键 ==================== + + /** + * C13 的确定性幂等键:{@code sha256("import:" + 表单 uuid + "|" + subject + "|" + 规范化值文档)} 前 40 位。 + * + *

批次串取表单坐标而不是文件字节摘要:字节摘要会让"改一个字重传整份文件"把未改动的那些行 + * 全部当新数据插一遍(1000 行里修 1 行 = 造出 999 条重复),那是不可接受的副作用。按表单坐标后 + * 键只随行内容变:同一张表里同样的行只落一次,改过的行才算新数据。也不含行号(中间插一行 + * 不该让后续所有行的键都变掉),更不含请求级随机串(那等于没有幂等)。见手册 §5 P128。

+ * + *

{@code "import:"} 前缀让导入产生的键与手工填报的客户端 uuid 键天然不相交({@code uk} 是库级的)。

+ */ + private static String idempotencyKey(String formUuid, Integer subjectId, JsonNode values) { + String canonical = "import:" + formUuid + "|" + (subjectId == null ? "" : subjectId) + "|" + values; + return digest(canonical.getBytes(StandardCharsets.UTF_8)).substring(0, IDEMPOTENCY_KEY_LENGTH); + } + + private static String digest(byte[] bytes) { + try { + return hex(MessageDigest.getInstance("SHA-256").digest(bytes)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 不可用", e); + } + } + + private static String hex(byte[] raw) { + StringBuilder sb = new StringBuilder(raw.length * 2); + for (byte b : raw) { + sb.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return sb.toString(); + } + + private static String trim(String raw) { + return raw == null ? null : raw.trim(); + } + + /** 文件侧的一行:物理行号(1 起,与用户在 Excel 里看到的行号一致)+ 列号到单元格文本 */ + private static final class SheetRow { + private final int rowNo; + private final Map cells; + + private SheetRow(int rowNo, Map cells) { + this.rowNo = rowNo; + this.cells = cells; + } + } +} diff --git a/src/test/java/com/yunhai/formcenter/service/AnswerImportIT.java b/src/test/java/com/yunhai/formcenter/service/AnswerImportIT.java new file mode 100644 index 0000000..a3b992a --- /dev/null +++ b/src/test/java/com/yunhai/formcenter/service/AnswerImportIT.java @@ -0,0 +1,391 @@ +package com.yunhai.formcenter.service; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.yunhai.formcenter.controller.api.FormAnswerReadController; +import com.yunhai.formcenter.entity.Answer; +import com.yunhai.formcenter.model.ImportModels; +import com.yunhai.formcenter.model.FormModels; +import com.yunhai.formcenter.repository.AnswerIndexRepository; +import com.yunhai.platform.apistd.BizException; +import com.yunhai.platform.utils.exception.ErrorCode; +import com.yunhai.platform.utils.model.BoaUser; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.transaction.annotation.Transactional; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * 批量导入(契约 v2 §3.4/§5.1/C13/C14,T18,验收 V60/V74)。 + * + *

主线只有一句话:导入的行与手工填报的行必须完全可比。所以断言都冲着"是不是同一条管线"去—— + * 值校验的报错文案与手工填报逐字相同、钉版与索引照写、限次照挡、被记录人走 {@code subjectId} + * 那一条 ALL 档判据(P116 裁决的代录口径),而不是导入侧另写的一套。

+ * + *

另三条是新机制自己的断言:C13 的确定性幂等键(同一文件重传逐行返回原 answerId、不落新行,P128)、 + * C14 的"表头先校验"(重名 label / 不存在的列 / 行数超上限都在写库之前整批拒)、坏行不静默 + * (逐行 ok/error,一行失败不影响其它行)。

+ */ +@SpringBootTest +@Import(AnswerTestSupport.TestGatewayConfiguration.class) +@Transactional +@DisplayName("批量导入 - 同一条提交管线与逐行结果(V60/V74,C13/C14)") +class AnswerImportIT extends AnswerTestSupport { + + /** text + searchable 的 select + number:够覆盖"必填 / 选项域 / 数字域"三类坏行 */ + private static final String SCHEMA = "{\"groups\":[{\"key\":\"base\",\"label\":\"基本信息\",\"fields\":[" + + "{\"key\":\"name\",\"label\":\"姓名\",\"type\":\"text\",\"required\":true}," + + "{\"key\":\"grade\",\"label\":\"年级\",\"type\":\"select\",\"required\":true,\"searchable\":true," + + "\"options\":[{\"label\":\"高二\",\"value\":\"g2\"},{\"label\":\"高三\",\"value\":\"g3\"}]}," + + "{\"key\":\"score\",\"label\":\"分数\",\"type\":\"number\"}]}]}"; + + private static final String SUBJECT_HEADER = "被记录人ID"; + private static final String NOTE = "说明:模板说明行,导入时按前缀跳过"; + private static final int TEACHER_ADMIN_ID = 20268001; + + @Autowired + private AnswerImportService importService; + + @Autowired + private FormAnswerReadController readController; + + @Autowired + private AnswerIndexRepository indexRows; + + // ==================== 模板 ==================== + + @Test + @DisplayName("模板两行表头:第一行=label + 固定第一列「被记录人ID」,第二行是给人看的说明(导入时按前缀跳过)") + void templateHeadIsLabelRowThenNoteRow() { + String uuid = publishForm("t18_template", SCHEMA); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + importService.template(SCH, uuid, out); + + List> read = readAllRows(out.toByteArray()); + assertThat(read).hasSize(2); + assertThat(read.get(0).values()).contains(SUBJECT_HEADER, "姓名", "年级", "分数"); + assertThat(read.get(1).get(0)).startsWith("说明:"); + // 说明行带必填与候选值提示,填表人不用回设计器对照 + assertThat(read.get(1).get(2)).contains("必填").contains("g2"); + } + + @Test + @DisplayName("未发布的表单没有可导入的字段定义:模板与导入都在入口拒(不给只有一列的空模板)") + void unpublishedFormHasNoTemplate() { + FormModels.FormCreateRequest create = new FormModels.FormCreateRequest(); + create.setCode("t18_draft_only"); + create.setName("草稿表"); + String uuid = defService.create(SCH, create, null).getFormUuid(); + + assertThatThrownBy(() -> importService.template(SCH, uuid, new ByteArrayOutputStream())) + .isInstanceOf(BizException.class).hasMessageContaining("尚未发布"); + assertThatThrownBy(() -> importService.importAnswers(adminTeacher(), SCH, uuid, + fileOf(xlsx(twoRowHead("姓名"), rows(row("甲")))))) + .isInstanceOf(BizException.class).hasMessageContaining("尚未发布"); + } + + // ==================== V60 同一条管线 ==================== + + @Test + @DisplayName("V60/V74:导入的两行落库带 form_uuid+钉住 version,searchable 字段照写索引(与手工填报同一管线)") + void importedRowsGoThroughTheSamePipeline() { + String code = "t18_import_ok"; + String uuid = managedTeacherForm(code); + + ImportModels.Result result = importService.importAnswers(adminTeacher(), SCH, uuid, + fileOf(xlsx(twoRowHead("姓名", "年级", "分数"), + rows(row("", "张三", "g2", "95"), row("", "李四", "g3", "88"))))); + + assertThat(result.getFailCount()).isZero(); + assertThat(result.getOkCount()).isEqualTo(2); + assertThat(result.getRows()).extracting(ImportModels.RowResult::getAnswerId).doesNotContainNull(); + + List stored = answers(code); + assertThat(stored).hasSize(2); + assertThat(stored.get(0).getFormUuid()).isEqualTo(uuid); + assertThat(stored.get(0).getVersion()).isEqualTo(1); + assertThat(stored).allSatisfy(row -> { + assertThat(row.getSubmitterId()).isEqualTo(TEACHER_ADMIN_ID); + // 自填归一(Bug#3 口径):被记录人留空 = 本人,不是 NULL + assertThat(row.getSubjectId()).isEqualTo(TEACHER_ADMIN_ID); + assertThat(row.getAnonIp()).isNull(); + assertThat(row.getValuesJson()).contains("grade"); + // 索引同管线:searchable 的 grade 必须进 answer_index,否则数据面筛不到导入的行 + assertThat(indexRows.findFieldKeysByAnswerId(row.getId())).contains("grade"); + }); + } + + @Test + @DisplayName("V74 坏行不静默:缺必填 / 选项域外 / 数字域外逐行给管线原话,好行照常写入;行号是文件物理行号") + void badRowsAreReportedPerRowWithoutAbortingTheBatch() { + String code = "t18_import_bad_rows"; + String uuid = managedTeacherForm(code); + + ImportModels.Result result = importService.importAnswers(adminTeacher(), SCH, uuid, + fileOf(xlsx(twoRowHead("姓名", "年级", "分数"), rows( + row("", "好行", "g2", "95"), + row("", "", "g2", ""), + row("", "乙", "g9", ""), + row("", "丙", "g2", "高分"))))); + + assertThat(result.getOkCount()).isEqualTo(1); + assertThat(result.getFailCount()).isEqualTo(3); + assertThat(result.getRows().get(0).getOk()).isTrue(); + assertThat(result.getRows().get(1).getAnswerId()).isNull(); + assertThat(result.getRows().get(1).getError()).contains("姓名").contains("必填"); + assertThat(result.getRows().get(2).getError()).contains("不在候选列表内"); + assertThat(result.getRows().get(3).getError()).contains("必须是数字"); + // 行号是文件里的物理行号(表头 1 行 + 说明 1 行 → 数据从第 3 行起) + assertThat(result.getRows()).extracting(ImportModels.RowResult::getRowNo).containsExactly(3, 4, 5, 6); + assertThat(answers(code)).hasSize(1); + } + + @Test + @DisplayName("被记录人列非整数:逐行拒(这一列不是自由文本),同批好行照常写入") + void nonNumericSubjectCellIsRejectedPerRow() { + String code = "t18_import_bad_subject"; + String uuid = managedTeacherForm(code); + + ImportModels.Result result = importService.importAnswers(adminTeacher(), SCH, uuid, + fileOf(xlsx(twoRowHead("姓名", "年级", "分数"), + rows(row("", "张三", "g2", "95"), row("abc", "甲", "g2", ""))))); + + assertThat(result.getOkCount()).isEqualTo(1); + assertThat(result.getRows().get(1).getError()).contains("被记录人ID").contains("整数"); + assertThat(answers(code)).hasSize(1); + } + + @Test + @DisplayName("C14+P116:填「被记录人」=代录,落 subject_id=该人、submitter 仍是操作者;非 ALL 档连端点都进不来") + void subjectColumnIsTheProxyPathGuardedByAllTier() { + String code = "t18_import_proxy"; + String uuid = managedTeacherForm(code); + int proxy = 20268101; + subjects.markSchool(proxy, SCH); + + ImportModels.Result result = importService.importAnswers(adminTeacher(), SCH, uuid, + fileOf(xlsx(twoRowHead("姓名", "年级", "分数"), + rows(row(String.valueOf(proxy), "甲", "g2", ""))))); + + assertThat(result.getOkCount()).isEqualTo(1); + Answer stored = answers(code).get(0); + assertThat(stored.getSubjectId()).isEqualTo(proxy); + assertThat(stored.getSubmitterId()).isEqualTo(TEACHER_ADMIN_ID); + + // 不是这张表 ALL 档的会话:控制器的 requireData 就拒,进不到服务(与导出/编辑同一把守卫) + BoaUser outsider = plainUser(1); + assertThatThrownBy(() -> readController.importAnswers(outsider, null, uuid, + fileOf(xlsx(twoRowHead("姓名", "年级"), rows(row("x", "g2")))))) + .isInstanceOf(BizException.class) + .satisfies(e -> assertThat(((BizException) e).getErrorCode()).isEqualTo(ErrorCode.FORBIDDEN)); + } + + // ==================== C13 确定性幂等键 ==================== + + @Test + @DisplayName("C13/P128:同一文件重传逐行返回原 answerId 且不落新行;改一个字只有那一行算新数据") + void sameFileReimportIsIdempotentRowByRow() { + String code = "t18_import_replay"; + String uuid = managedTeacherForm(code); + List> head = twoRowHead("姓名", "年级", "分数"); + byte[] same = xlsx(head, rows(row("", "甲", "g2", "95"), row("", "乙", "g3", "88"))); + + ImportModels.Result first = importService.importAnswers(adminTeacher(), SCH, uuid, fileOf(same)); + ImportModels.Result second = importService.importAnswers(adminTeacher(), SCH, uuid, fileOf(same)); + + assertThat(first.getOkCount()).isEqualTo(2); + assertThat(ids(second.getRows())).isEqualTo(ids(first.getRows())); + assertThat(answers(code)).hasSize(2); + + byte[] edited = xlsx(head, rows(row("", "甲", "g2", "95"), row("", "乙改", "g3", "88"))); + ImportModels.Result third = importService.importAnswers(adminTeacher(), SCH, uuid, fileOf(edited)); + + assertThat(third.getOkCount()).isEqualTo(2); + assertThat(third.getRows().get(0).getAnswerId()).isEqualTo(first.getRows().get(0).getAnswerId()); + assertThat(third.getRows().get(1).getAnswerId()).isNotEqualTo(first.getRows().get(1).getAnswerId()); + assertThat(answers(code)).hasSize(3); + } + + @Test + @DisplayName("限次对导入同样生效(同一把管线闸):同 subject 的第二行被限次拒") + void fillLimitAppliesToImportedRowsToo() { + String code = "t18_import_quota"; + String uuid = managedTeacherForm(code); + config(SCH, uuid, fillOnce()); + + ImportModels.Result result = importService.importAnswers(adminTeacher(), SCH, uuid, + fileOf(xlsx(twoRowHead("姓名", "年级", "分数"), + rows(row("", "甲", "g2", ""), row("", "乙", "g2", ""))))); + + assertThat(result.getOkCount()).isEqualTo(1); + assertThat(result.getRows().get(1).getError()).contains("限填一次"); + assertThat(answers(code)).hasSize(1); + } + + // ==================== 整批拒的三件 ==================== + + @Test + @DisplayName("C14:表头有当前发布版里不存在的列 → 整批拒且零写入(不做忽略那一列的静默降级)") + void unknownHeaderColumnRejectsWithoutWritingAnything() { + String code = "t18_import_header_unknown"; + String uuid = managedTeacherForm(code); + + assertThatThrownBy(() -> importService.importAnswers(adminTeacher(), SCH, uuid, + fileOf(xlsx(twoRowHead("姓名", "年级", "手机"), rows(row("甲", "g2")))))) + .isInstanceOf(BizException.class).hasMessageContaining("不存在的列").hasMessageContaining("手机"); + assertThat(answers(code)).isEmpty(); + } + + @Test + @DisplayName("C14:重名 label 在模板与导入两侧都拒,并提示先在设计器改 label") + void duplicateLabelIsRejectedOnBothSides() { + String code = "t18_import_dup_label"; + String uuid = publishForm(code, SCHEMA.replace("\"label\":\"年级\"", "\"label\":\"姓名\"")); + config(SCH, uuid, audience(Arrays.asList("STUDENT", "TEACHER"))); + grantFormAdmin(uuid, TEACHER_ADMIN_ID, SCH); + + assertThatThrownBy(() -> importService.template(SCH, uuid, new ByteArrayOutputStream())) + .isInstanceOf(BizException.class).hasMessageContaining("重复").hasMessageContaining("先在设计器改 label"); + assertThatThrownBy(() -> importService.importAnswers(adminTeacher(), SCH, uuid, + fileOf(xlsx(twoRowHead("姓名", "姓名"), rows(row("甲", "g2")))))) + .isInstanceOf(BizException.class).hasMessageContaining("重复"); + assertThat(answers(code)).isEmpty(); + } + + @Test + @DisplayName("C14:行上限 1000 是整批拒不是截断,文案给实际行数;一件写库都不发生") + void rowCapRejectsTheWholeBatchInsteadOfTruncating() { + String code = "t18_import_cap"; + String uuid = managedTeacherForm(code); + List> big = new ArrayList<>(); + for (int i = 0; i < 1001; i++) { + big.add(row("", "行" + i, "g2", "")); + } + + assertThatThrownBy(() -> importService.importAnswers(adminTeacher(), SCH, uuid, + fileOf(xlsx(twoRowHead("姓名", "年级", "分数"), big)))) + .isInstanceOf(BizException.class).hasMessageContaining("上限 1000").hasMessageContaining("1001"); + assertThat(answers(code)).isEmpty(); + } + + @Test + @DisplayName("附件类字段不接受导入值:逐行点名批量导入不支持上传,不静默丢那一列") + void attachmentCellsAreRejectedPerRow() { + String code = "t18_import_attach"; + String uuid = managedTeacherForm(code, "{\"groups\":[{\"key\":\"base\",\"label\":\"基本信息\",\"fields\":[" + + "{\"key\":\"cert\",\"label\":\"证书\",\"type\":\"image\"}]}]}"); + + ImportModels.Result result = importService.importAnswers(adminTeacher(), SCH, uuid, + fileOf(xlsx(twoRowHead("证书"), rows(row("", "a.jpg"))))); + + assertThat(result.getFailCount()).isEqualTo(1); + assertThat(result.getRows().get(0).getError()).contains("附件类").contains("批量导入不支持"); + assertThat(answers(code)).isEmpty(); + } + + // ==================== 脚手架 ==================== + + /** 一张"面向学生+教师、操作者是 ALL 档(form_admin)"的已发布表;人群含 TEACHER 才让自填过得了 audience 门 */ + private String managedTeacherForm(String code) { + return managedTeacherForm(code, SCHEMA); + } + + private String managedTeacherForm(String code, String schema) { + String uuid = publishForm(code, schema); + config(SCH, uuid, audience(Arrays.asList("STUDENT", "TEACHER"))); + grantFormAdmin(uuid, TEACHER_ADMIN_ID, SCH); + return uuid; + } + + /** 导入操作人:教师身份(在人群内)+ 该表 ALL 档(form_admin)——自填与代录两条路都走得通 */ + private BoaUser adminTeacher() { + return user(TEACHER_ADMIN_ID, SCH, 1); + } + + private List answers(String code) { + return answerRepository.findBySchIdAndAppCodeAndCodeOrderByIdAsc(SCH, "form-center", code); + } + + private static FormModels.FormConfigRequest audience(List audience) { + FormModels.FormConfigRequest cfg = new FormModels.FormConfigRequest(); + cfg.setAudience(audience); + return cfg; + } + + private static FormModels.FormConfigRequest fillOnce() { + FormModels.FormConfigRequest cfg = new FormModels.FormConfigRequest(); + cfg.setFillLimit(1); + return cfg; + } + + /** 两行表头(第一行 label、第二行说明),与 {@code AnswerImportService#headOf} 同形 */ + private static List> twoRowHead(String... labels) { + List> head = new ArrayList<>(); + head.add(Arrays.asList(SUBJECT_HEADER, NOTE)); + for (String label : labels) { + head.add(Arrays.asList(label, NOTE + "|" + label)); + } + return head; + } + + private static List row(Object... cells) { + return new ArrayList<>(Arrays.asList(cells)); + } + + @SafeVarargs + private static List> rows(List... dataRows) { + return new ArrayList<>(Arrays.asList(dataRows)); + } + + /** 写一份 xlsx:head 为两行表头(EasyExcel 自己写两行),dataRows 只含真数据行 */ + private static byte[] xlsx(List> head, List> dataRows) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + EasyExcel.write(out).head(head).sheet("导入").doWrite(dataRows); + return out.toByteArray(); + } + + private static MockMultipartFile fileOf(byte[] bytes) { + return new MockMultipartFile("file", "import.xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", bytes); + } + + private static List ids(List rows) { + List ids = new ArrayList<>(); + for (ImportModels.RowResult row : rows) { + ids.add(row.getAnswerId()); + } + return ids; + } + + /** 读回 xlsx(headRowNumber(0)=表头也按行返回),断言模板形状用 */ + private static List> readAllRows(byte[] bytes) { + List> all = new ArrayList<>(); + EasyExcel.read(new ByteArrayInputStream(bytes), new AnalysisEventListener>() { + @Override + public void invoke(Map row, AnalysisContext context) { + all.add(new LinkedHashMap<>(row)); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // 收满即止 + } + }).sheet().headRowNumber(0).doRead(); + return all; + } +}