diff --git a/api/src/org/labkey/api/data/DataColumn.java b/api/src/org/labkey/api/data/DataColumn.java index 7b66f384a75..0f90f9b2012 100644 --- a/api/src/org/labkey/api/data/DataColumn.java +++ b/api/src/org/labkey/api/data/DataColumn.java @@ -191,6 +191,12 @@ protected ColumnInfo getDisplayField(@NotNull ColumnInfo col, boolean withLookup return null==display ? col : display; } + @Override + public void setWithLookup(boolean withLookup) + { + _displayColumn = withLookup ? getDisplayField(_boundColumn, true) : _boundColumn; + } + @Override public String toString() { diff --git a/api/src/org/labkey/api/data/DisplayColumn.java b/api/src/org/labkey/api/data/DisplayColumn.java index a954198c9c1..659a1b69dda 100644 --- a/api/src/org/labkey/api/data/DisplayColumn.java +++ b/api/src/org/labkey/api/data/DisplayColumn.java @@ -1232,6 +1232,11 @@ public void setRequiresHtmlFiltering(boolean requiresHtmlFiltering) _requiresHtmlFiltering = requiresHtmlFiltering; } + public void setWithLookup(boolean withLookup) + { + // subclasses override as needed + } + public void setLinkTarget(String linkTarget) { _linkTarget = linkTarget; diff --git a/api/src/org/labkey/api/util/PageFlowUtil.java b/api/src/org/labkey/api/util/PageFlowUtil.java index 520a6c2d171..0bcc365adce 100644 --- a/api/src/org/labkey/api/util/PageFlowUtil.java +++ b/api/src/org/labkey/api/util/PageFlowUtil.java @@ -2697,6 +2697,16 @@ private static boolean shouldEscapeForExport(@NotNull String value) return StringUtils.containsAny(value,",\""); } + /// Generate one row of tab-delimited output using RFC 4180 quoting rules. + /// Fields containing tabs, newlines, or double quotes are enclosed in double quotes, + /// with embedded double quotes escaped by doubling. + public static String joinValuesWithTabs4180(@NotNull List values) + { + return values.stream() + .map(value -> null == value ? "" : StringUtils.containsAny(value, "\t\n\r\"") ? "\"" + Strings.CS.replace(value, "\"", "\"\"") + "\"" : value) + .collect(Collectors.joining("\t")); + } + static final String FIELD_ENCODED_PREFIX = "%_"; diff --git a/query/src/org/labkey/query/QueryModule.java b/query/src/org/labkey/query/QueryModule.java index 8d47e150389..22eca20a809 100644 --- a/query/src/org/labkey/query/QueryModule.java +++ b/query/src/org/labkey/query/QueryModule.java @@ -402,7 +402,8 @@ public Set getSchemaNames() RolapReader.RolapTest.class, RolapTestCase.class, SelectRowsStreamHack.TestCase.class, - ServerManager.TestCase.class + ServerManager.TestCase.class, + SqlController.TestCase.class ); } diff --git a/query/src/org/labkey/query/controllers/QueryMcp.java b/query/src/org/labkey/query/controllers/QueryMcp.java index 07e28cee290..3a4391b16e1 100644 --- a/query/src/org/labkey/query/controllers/QueryMcp.java +++ b/query/src/org/labkey/query/controllers/QueryMcp.java @@ -44,9 +44,9 @@ import org.labkey.api.query.UserSchema; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.query.QueryServiceImpl; import org.labkey.api.view.NotFoundException; import org.labkey.api.writer.ContainerUser; +import org.labkey.query.QueryServiceImpl; import org.labkey.query.sql.SqlParser; import org.springframework.ai.chat.model.ToolContext; import org.springframework.ai.mcp.annotation.McpResource; @@ -54,12 +54,14 @@ import org.springframework.ai.tool.annotation.ToolParam; import java.io.IOException; +import java.io.StringWriter; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.TreeMap; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.labkey.api.util.StringUtilsLabKey.pluralize; public class QueryMcp implements McpService.McpImpl { @@ -210,6 +212,54 @@ String validateCalculatedColumnExpression( } } + + @Tool(description = + "Execute a LabKey SQL query and return results as tab-separated values (RFC 4180 TSV). " + + "Use this to inspect actual query results while writing or debugging SQL. " + + "Prefer validateSQL when you only need to check syntax without running the query. " + + "Returns at most 100 rows; use offset and limit to page through larger result sets. " + + "Response format: a header row of column names, then one data row per newline, fields tab-separated. " + + "Fields containing tabs, newlines, or double-quotes are RFC 4180 quoted. " + + "On SQL error, the error message is returned as plain text rather than throwing. " + + "For data analysis or bulk retrieval, use the LabKey Python or R client APIs instead of this tool. " + + "**Important** This tool does not yet support queries with named parameters.") + @RequiresPermission(ReadPermission.class) + String executeSQL( + ToolContext toolContext, + @ToolParam(description = "Fully qualified schema name as it would appear in SQL e.g. Study or \"Study\".\"Datasets\"") String schemaName, + @ToolParam(description = "LabKey SQL to execute") String sql, + @ToolParam(description = "Rows to skip before returning results.", required=false) Integer offset, + @ToolParam(description = "Number of rows to return (limit <= 100)", required=false) Integer limit + ) + { + var cu = getContext(toolContext); + var schema = DefaultSchema.get(cu.getUser(), cu.getContainer(), getSchemaKey(schemaName)); + if (!(schema instanceof UserSchema userSchema)) + return "Could not find schema " + schemaName; + + offset = null==offset ? 0 : offset < 0 ? 0 : offset; + limit = (limit == null || limit < 0) ? 100 : Math.min(100, limit); + var execute = new SqlController.SqlExecute(cu, userSchema, sql) + .page(offset, limit) + .truncation(500, "…[truncated]"); + + try + { + StringWriter sw = new StringWriter(2000); + SqlController.SqlExecute.ExecuteResult result = execute.execute(sw); + String message = "\n-- " + pluralize(result.rows(), "row", "rows") + " returned"; + if (result.complete()) + message += "."; + else + message += ", more may be available (use offset and limit to page)."; + return sw + message; + } + catch (Exception x) + { + return x.getMessage() != null ? x.getMessage() : x.getClass().getSimpleName(); + } + } + /* For now, list all schemas. CONSIDER support incremental querying. */ public static Map _listAllSchemas(DefaultSchema root) { diff --git a/query/src/org/labkey/query/controllers/SqlController.java b/query/src/org/labkey/query/controllers/SqlController.java index ffa6f8f4ba1..94d4b1f020f 100644 --- a/query/src/org/labkey/query/controllers/SqlController.java +++ b/query/src/org/labkey/query/controllers/SqlController.java @@ -16,34 +16,65 @@ package org.labkey.query.controllers; import com.fasterxml.jackson.annotation.JsonAnySetter; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; import org.labkey.api.action.Marshal; import org.labkey.api.action.Marshaller; import org.labkey.api.action.ReadOnlyApiAction; import org.labkey.api.action.SpringActionController; import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.ResultSetRowMapFactory; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DisplayColumn; import org.labkey.api.data.JdbcType; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.RenderContext; import org.labkey.api.data.Results; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.IPropertyValidator; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.ontology.Unit; +import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.QueryParseException; -import org.labkey.api.query.QuerySchema; import org.labkey.api.query.QueryService; import org.labkey.api.query.SchemaKey; import org.labkey.api.query.UserSchema; import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.User; import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.util.DateUtil; +import org.labkey.api.util.JunitUtil; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.ViewServlet; +import org.labkey.api.writer.ContainerUser; import org.springframework.beans.PropertyValue; +import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.validation.BindException; -import jakarta.servlet.ServletException; - import java.io.IOException; -import java.io.PrintWriter; -import java.math.BigDecimal; +import java.io.StringWriter; +import java.io.Writer; import java.sql.SQLException; -import java.util.Date; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -53,6 +84,10 @@ public class SqlController extends SpringActionController { private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(SqlController.class); + static final char RS = '\036'; // ASCII Record Separator + static final char US = '\037'; // ASCII Unit Separator + static final char BS = '\010'; // ASCII Backspace (ditto marker) + public SqlController() { setActionResolver(_actionResolver); @@ -70,14 +105,41 @@ public void set(String name, Object value) } } + public enum Format + { + split("text/plain"), // response that can be parsed using String.split(), configure with 'sep' and 'eol' + + compact("text/plain"), // same as split, but with ditto markers (cheap way to compress and save space on client) + + // Tab separated (RFC 4180) + // - Wrap a field in double quotes if it contains a tab, newline, or double quote + // - Escape double quotes by doubling them: " becomes "" + // - Fields that don't contain special characters are left unquoted + tsv("text/tab-separated-values"); + + final String contentType; + + Format(String contentType) + { + this.contentType = contentType; + } + + String getContentType() + { + return contentType; + } + // We could support CSV here, I can't think of a scenario for this API where this would be preferable + // csv // Comma separated, google sheets style quoting (see PageFlowUtil.joinValuesToStringForExport()) + } + public static class SqlForm { + private Format format = Format.split; private Double apiVersion = null; private String schema; private String sql; private String sep = null; private String eol = null; - private boolean compact = false; private final Parameters parameters = new Parameters(); public Double getApiVersion() @@ -130,9 +192,19 @@ public Map getParameterMap() return parameters.map; } + public Format getFormat() + { + return format; + } + + public void setFormat(Format format) + { + this.format = format; + } + public String getSep() { - return null!=sep ? sep : compact ? "\u001f" : "\t"; + return null!=sep ? sep : format==Format.compact ? String.valueOf(US) : "\t"; } public void setSep(String sep) @@ -143,7 +215,7 @@ public void setSep(String sep) public String getEol() { - return null!=eol ? eol : compact ? "\u001e" : "\t"; + return null!=eol ? eol : format==Format.compact ? String.valueOf(RS) : "\n"; } public void setEol(String eol) @@ -154,19 +226,293 @@ public void setEol(String eol) public boolean isCompact() { - return compact; + return Format.compact == format; } public void setCompact(boolean compact) { - this.compact = compact; + if (compact) + this.format = Format.compact; } } + public static class SqlExecute + { + final UserSchema schema; + final String sql; + Format format; + String sep, eol; + Integer offset; + Integer limit; + Integer maxStringLength = -1; + String truncationSuffix = ""; + Map parameterMap = Map.of(); + + JdbcType[] types; + Unit[] units; + DisplayColumn[] dcs; + RenderContext renderCtx; + ResultSetRowMapFactory rowMapFactory; + + private final ContainerUser ctx; + + public SqlExecute(ContainerUser ctx, UserSchema schema, String sql) + { + this.ctx = ctx; + this.schema = schema; + this.sql = sql; + this.format = Format.tsv; + } + + SqlExecute page(Integer offset, int limit) + { + this.offset = offset; + this.limit = limit; + return this; + } + + SqlExecute truncation(int max, String suffix) + { + maxStringLength = max; + truncationSuffix = suffix; + return this; + } + + SqlExecute parameters(Map parameters) + { + this.parameterMap = new HashMap<>(parameters); + return this; + } + + SqlExecute format(Format format, String sep, String eol) + { + this.format = format; + this.sep = sep; + this.eol = eol; + return this; + } + void initWriter(Results rs) throws SQLException + { + final int count = rs.getMetaData().getColumnCount(); + types = new JdbcType[count]; + units = new Unit[count]; + dcs = new DisplayColumn[count]; + + for (int column = 1; column <= count; column++) + { + int index = column - 1; + types[index] = JdbcType.valueOf(rs.getMetaData().getColumnType(column)); + ColumnInfo col = rs.getColumn(column); + units[index] = col.getDisplayUnit(); + DisplayColumn dc = col.getDisplayColumnFactory().createRenderer(col); + dc.setWithLookup(false); + dc.setFormatString(null); + dc.setTsvFormatString(null); + dc.setRequiresHtmlFiltering(false); + dcs[index] = dc; + } + ViewContext viewContext = new ViewContext(); + viewContext.setUser(ctx.getUser()); + viewContext.setContainer(ctx.getContainer()); + renderCtx = new RenderContext(viewContext); + renderCtx.setResults(rs); + rowMapFactory = ResultSetRowMapFactory.create(rs); + } + + void getStringData(Results rs, ArrayList out) throws SQLException + { + out.clear(); + renderCtx.setRow(rowMapFactory.getRowMap(rs)); + for (DisplayColumn dc : dcs) + { + String string = dc.getTsvFormattedValue(renderCtx); + if (maxStringLength > 0 && null != string && string.length() > maxStringLength) + { + string = string.substring(0,maxStringLength) + truncationSuffix; + } + out.add(string); + } + } + + int writeResults_text(Writer out, Results rs, int limit, String sep, String eol) throws IOException, SQLException + { + initWriter(rs); + final int columnCount = rs.getMetaData().getColumnCount(); + + // meta-meta-data + out.write("18.2"+sep+"name"+sep+"jdbcType"+eol); + + for (int column = 1; column <= columnCount; column++) + { + out.write(rs.getColumn(column).getName()); + out.write(column == columnCount ? eol : sep); + } + + for (int column = 1; column <= columnCount; column++) + { + int index = column-1; + out.write(types[index].name()); + out.write(column == columnCount ? eol : sep); + } + + ArrayList values = new ArrayList<>(columnCount); + + int count = 0; + while (--limit >= 0 && rs.next()) + { + getStringData(rs, values); + for (int index = 0; index < columnCount; index++) + { + String s = values.get(index); + if (null != s) + out.write(s); + out.write(index == columnCount - 1 ? eol : sep); + } + count++; + } + out.flush(); + return count; + } + + /** + * try to generate a more compact representation + * + * the value 0x1A SUB means same value as previous row + * truncate trailing time from date-only values + * + * Consider disambiguating sql NULL and empty string. NULL is more common so using a shorter + * encoding for NULL and longer for empty string makes sense (e.g. "\u0000") + * + * TODO: binary/blob length prefixed encoding + * + * Note that while writeResults_text tries to not generate extra Strings to GC, I think the actual PrintWriter + * implementation is generating strings inside out.write(). So this is probably not much different from a GC + * perspective. + */ + int writeResults_compact(Writer out, Results rs, int limit, String sep, String eol) throws IOException, SQLException + { + initWriter(rs); + final int columnCount = rs.getMetaData().getColumnCount(); + + // meta-meta-data + out.write("18.2"+sep+"name"+sep+"jdbcType"+eol); + + for (int column = 1; column <= columnCount; column++) + { + out.write(rs.getColumn(column).getName()); + out.write(column == columnCount ? eol : sep); + } + + for (int index = 0; index < columnCount; index++) + { + out.write(types[index].name()); + out.write(index == columnCount-1 ? eol : sep); + } + + String DITTO = String.valueOf(BS); + ArrayList prev = new ArrayList<>(columnCount); + ArrayList row = new ArrayList<>(columnCount); + + int count = 0; + while (--limit >= 0 && rs.next()) + { + getStringData(rs, row); + + for (int index = 0; index < columnCount; index++) + { + String s = row.get(index); + if (null != s && !s.isEmpty()) + { + if (index < prev.size() && s.equals(prev.get(index))) + out.write(DITTO); + else + out.write(s); + } + out.write(index == columnCount - 1 ? eol : sep); + } + ArrayList t = prev; + prev = row; + row = t; + count++; + } + out.flush(); + return count; + } + + /// export a Result set using RFC4180 formatting + /// use PageFlowUtil.joinValuesWithTabs4180 + int writeResults_tsv(Writer out, Results rs, int limit) throws IOException, SQLException + { + initWriter(rs); + final int columnCount = rs.getMetaData().getColumnCount(); + + List names = new ArrayList<>(columnCount); + for (int column = 1; column <= columnCount; column++) + names.add(rs.getColumn(column).getName()); + out.write(PageFlowUtil.joinValuesWithTabs4180(names)); + out.write('\n'); + + ArrayList values = new ArrayList<>(columnCount); + + int count = 0; + while (--limit >= 0 && rs.next()) + { + getStringData(rs, values); + out.write(PageFlowUtil.joinValuesWithTabs4180(values)); + out.write('\n'); + count++; + } + out.flush(); + return count; + } + + public record ExecuteResult(int rows, boolean complete) {} + + public ExecuteResult execute(Writer out) throws SQLException, IOException + { + schema.checkCanReadSchema(); + var builder = QueryService.get().getSelectBuilder(schema, sql, true); + if (null != offset) + builder.offset(offset); + int maxPrint = Integer.MAX_VALUE; + if (null != limit && limit < Integer.MAX_VALUE) + { + builder.maxRows(limit + 1); + maxPrint = limit; + } + + try (Results rs = builder.select(false, parameterMap)) + { + int count; + boolean complete = true; + switch (format) + { + case tsv: + count = writeResults_tsv(out, rs, maxPrint); + break; + case split: + count = writeResults_text(out, rs, maxPrint, sep, eol); + break; + case compact: + count = writeResults_compact(out, rs, maxPrint, sep, eol); + break; + default: + throw new IllegalArgumentException("Unknown format: " + format); + } + while (rs.next()) + complete = false; + return new ExecuteResult(count, complete); + } + } + } + + /// Execute a LabKey SQL query and return results as plain text. Designed for lightweight programmatic access without the overhead of QueryView/JSON API responses. + /// This action routes value rendering through DisplayColumn.getTsvFormattedValue() for correct + /// type dispatch (e.g. dates as ISO-8601, multi-value columns via their DisplayColumn subclass). @RequiresPermission(ReadPermission.class) @Marshal(Marshaller.Jackson) - public class ExecuteAction extends ReadOnlyApiAction + public static class ExecuteAction extends ReadOnlyApiAction { @Override public Object execute(SqlForm form, BindException errors) throws ServletException, SQLException, IOException @@ -175,7 +521,7 @@ public Object execute(SqlForm form, BindException errors) throws ServletExceptio SchemaKey schemaKey = null == schemaString ? new SchemaKey(null,"core") : SchemaKey.decode(schemaString); { // spring binding doesn't handle this form very well ( - if (!StringUtils.contains(getViewContext().getRequest().getContentType(),"json")) + if (!Strings.CS.contains(getViewContext().getRequest().getContentType(),"json")) { // white space is broken for separators, so rebind just in case if (StringUtils.isNotEmpty((String)getProperty("sep"))) @@ -184,7 +530,7 @@ public Object execute(SqlForm form, BindException errors) throws ServletExceptio form.setEol((String)getProperty("eol")); for (PropertyValue pv : getPropertyValues().getPropertyValues()) { - if (StringUtils.startsWith(pv.getName(),"parameters.")) + if (Strings.CS.startsWith(pv.getName(),"parameters.")) { String name = pv.getName().substring("parameters.".length()); Object value = pv.getValue(); @@ -194,31 +540,27 @@ public Object execute(SqlForm form, BindException errors) throws ServletExceptio } } - QuerySchema schema = DefaultSchema.get(getUser(), getContainer()); - for (String s : schemaKey.getParts()) + var schema = DefaultSchema.get(getUser(), getContainer(), schemaKey); + if (!(schema instanceof UserSchema userSchema)) { - schema = schema.getSchema(s); - if (null == schema) - { - errors.reject(ERROR_MSG, "schema not found: " + form.getSchema()); - return null; - } + errors.reject(ERROR_MSG, "Schema not found"); + return null; } - ((UserSchema)schema).checkCanReadSchema(); if (StringUtils.isEmpty(form.getSql())) { - errors.reject(ERROR_MSG, "no sql provided"); + errors.reject(ERROR_MSG, "No sql provided"); return null; } - try (Results rs = QueryService.get().getSelectBuilder(schema, form.getSql(), true).select(false, form.getParameterMap())) + var format = form.getFormat(); + getViewContext().getResponse().setContentType(format.getContentType()); + try { - getViewContext().getResponse().setContentType("text/plain"); - if (form.compact) - writeResults_compact(getViewContext().getResponse().getWriter(), rs, form.getSep(),form.getEol()); - else - writeResults_text(getViewContext().getResponse().getWriter(), rs, form.getSep(),form.getEol()); + new SqlExecute(getViewContext(), userSchema, form.getSql()) + .format(format, form.getSep(), form.getEol()) + .parameters(form.getParameterMap()) + .execute(getViewContext().getResponse().getWriter()); } catch (QueryParseException x) { @@ -229,214 +571,327 @@ public Object execute(SqlForm form, BindException errors) throws ServletExceptio } - void writeResults_text(PrintWriter out, Results rs, String sep, String eol) throws SQLException + public static class TestCase extends Assert { - final int count = rs.getMetaData().getColumnCount(); - final boolean serializeDateAsNumber=false; + private static final String FOLDER_NAME = "sqlControllerTest"; + private static final String LIST_NAME = "SqlTestList"; - // meta-meta-data - out.write("18.2"+sep+"name"+sep+"jdbcType"+eol); + private Container _folder; - for (int i = 1; i <= count; i++) + @Before + public void setUp() throws Exception { - out.write(rs.getColumn(i).getName()); - out.write(i == count ? eol : sep); + tearDown(); + Assume.assumeTrue("Requires list module", ListService.get() != null); + + User user = TestContext.get().getUser(); + _folder = ContainerManager.ensureContainer(JunitUtil.getTestContainer().getPath() + "/" + FOLDER_NAME, user); + + ListDefinition list = ListService.get().createList(_folder, LIST_NAME, ListDefinition.KeyType.AutoIncrementInteger); + list.setKeyName("Key"); + list.getDomain().addProperty(new PropertyStorageSpec("Name", JdbcType.VARCHAR)); + list.getDomain().addProperty(new PropertyStorageSpec("Age", JdbcType.INTEGER)); + list.getDomain().addProperty(new PropertyStorageSpec("Score", JdbcType.DOUBLE)); + + if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + DomainProperty tagsProp = list.getDomain().addProperty(new PropertyStorageSpec("Tags", JdbcType.VARCHAR)); + tagsProp.setRangeURI(PropertyType.MULTI_CHOICE.getTypeUri()); + IPropertyValidator tcValidator = PropertyService.get().createValidator("urn:lsid:labkey.com:PropertyValidator:textchoice"); + tcValidator.setName("Text Choice Validator"); + tcValidator.setExpressionValue("Red|Green|Blue"); + tagsProp.addValidator(tcValidator); + } + + list.save(user); + + TableInfo table = DefaultSchema.get(user, _folder).getSchema("lists").getTable(LIST_NAME, null); + assertNotNull("List table not found", table); + + BatchValidationException errors = new BatchValidationException(); + table.getUpdateService().insertRows(user, _folder, List.of( + CaseInsensitiveHashMap.of("Name", "Alice", "Age", 30, "Score", 95.5, "Tags", List.of("Red", "Green")), + CaseInsensitiveHashMap.of("Name", "Bob", "Age", 30, "Score", 87.3, "Tags", List.of("Blue")), + CaseInsensitiveHashMap.of("Name", "Carol", "Age", 35, "Score", 91.0, "Tags", List.of("Red", "Blue", "Green")) + ), errors, null, null); + if (errors.hasErrors()) + fail(errors.getRowErrors().get(0).toString()); } - // pull types from ResultSetMetaData, not ColumnInfo - JdbcType[] types = new JdbcType[count + 1]; - for (int i = 1; i <= count; i++) + @After + public void tearDown() { - JdbcType jdbc = JdbcType.valueOf(rs.getMetaData().getColumnType(i)); - types[i] = jdbc; - out.write(jdbc.name()); - out.write(i == count ? eol : sep); + Container folder = ContainerManager.getForPath(JunitUtil.getTestContainer().getPath() + "/" + FOLDER_NAME); + if (folder != null) + ContainerManager.deleteAll(folder, TestContext.get().getUser()); + _folder = null; } - while (rs.next()) + private MockHttpServletResponse executeSql(String schemaName, String sql, Format format) throws Exception { - for (int column = 1; column <= count; column++) + ActionURL url = new ActionURL("sql", "execute", _folder); + if (schemaName != null) + url.addParameter("schemaName", schemaName); + if (sql != null) + url.addParameter("sql", sql); + if (null != format) + url.addParameter("format", format.name()); + return ViewServlet.GET(url, TestContext.get().getUser(), null); + } + + @Test + public void testExecute_basic() throws Exception + { + MockHttpServletResponse response = executeSql("lists", + "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", Format.split); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String[] tokens = content.split("[\t\n]"); + + // Header: meta-meta-data (3) + column names (3) + types (3) = 9 + // Data: 3 rows * 3 columns = 9 + assertTrue("Expected at least 18 tokens, got " + tokens.length, tokens.length >= 18); + + // Meta-meta-data + assertEquals("18.2", tokens[0]); + assertEquals("name", tokens[1]); + assertEquals("jdbcType", tokens[2]); + + // Column names + assertEquals("Name", tokens[3]); + assertEquals("Age", tokens[4]); + assertEquals("Score", tokens[5]); + + // JDBC types + assertEquals("VARCHAR", tokens[6]); + assertEquals("INTEGER", tokens[7]); + assertEquals("DOUBLE", tokens[8]); + + // Data rows ordered by Name + assertEquals("Alice", tokens[9]); + assertEquals("30", tokens[10]); + assertEquals("95.5", tokens[11]); + assertEquals("Bob", tokens[12]); + assertEquals("30", tokens[13]); + assertEquals("87.3", tokens[14]); + assertEquals("Carol", tokens[15]); + assertEquals("35", tokens[16]); + assertEquals("91.0", tokens[17]); + } + + @Test + public void testExecute() throws Exception + { + if (!CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) { - // let's try to avoid tons of inspection if possible, and allocating tons of objects - // handle the most common types - printValue: - { - switch (types[column]) - { - case TINYINT: - case SMALLINT: - case INTEGER: - { - int i = rs.getInt(column); - if (!rs.wasNull()) - out.print(i); - break printValue; - } - case CHAR: - case VARCHAR: - { - String s = rs.getString(column); - if (null != s) - out.write(s); - break printValue; - } - case DOUBLE: - case REAL: - { - double d = rs.getDouble(column); - if (!rs.wasNull()) - out.print(d); - break printValue; - } - case TIMESTAMP: - { - Date date = rs.getTimestamp(column); - if (null != date) - { - if (serializeDateAsNumber) - out.print(date.getTime()); - else - //out.write(DateUtil.formatJsonDateTime(date)); - out.write(DateUtil.formatIsoDateShortTime(date)); - } - break printValue; - } - case BOOLEAN: - { - boolean b = rs.getBoolean(column); - if (!rs.wasNull()) - out.write(b ? '1' : '0'); - break printValue; - } - case DECIMAL: - { - BigDecimal dec = rs.getBigDecimal(column); - if (null != dec) - out.write(dec.toPlainString()); - break printValue; - } - default: - { - String obj = rs.getString(column); - if (null != obj) - out.write(obj); - break printValue; - } - } - } - out.write(column == count ? eol : sep); + testExecute_basic(); + return; } + + MockHttpServletResponse response = executeSql("lists", + "SELECT Name, Age, Score, Tags FROM " + LIST_NAME + " ORDER BY Name", Format.split); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String[] tokens = content.split("[\t\n]"); + + // Header: meta-meta-data (3) + column names (4) + types (4) = 11 + // Data: 3 rows * 4 columns = 12 + assertTrue("Expected at least 23 tokens, got " + tokens.length, tokens.length >= 23); + + // Meta-meta-data + assertEquals("18.2", tokens[0]); + assertEquals("name", tokens[1]); + assertEquals("jdbcType", tokens[2]); + + // Column names + assertEquals("Name", tokens[3]); + assertEquals("Age", tokens[4]); + assertEquals("Score", tokens[5]); + assertEquals("Tags", tokens[6]); + + // JDBC types + assertEquals("VARCHAR", tokens[7]); + assertEquals("INTEGER", tokens[8]); + assertEquals("DOUBLE", tokens[9]); + assertEquals("ARRAY", tokens[10]); + + // Data rows ordered by Name (4 columns per row) + assertEquals("Alice", tokens[11]); + assertEquals("30", tokens[12]); + assertEquals("95.5", tokens[13]); + assertTrue("Alice Tags", tokens[14].contains("Red") && tokens[14].contains("Green")); + assertEquals("Bob", tokens[15]); + assertEquals("30", tokens[16]); + assertEquals("87.3", tokens[17]); + assertTrue("Bob Tags", tokens[18].contains("Blue")); + assertEquals("Carol", tokens[19]); + assertEquals("35", tokens[20]); + assertEquals("91.0", tokens[21]); + assertTrue("Carol Tags", tokens[22].contains("Red") && tokens[22].contains("Blue") && tokens[22].contains("Green")); } - out.flush(); - } - /** - * try to generate a more compact representation - * - * the value 0x1A SUB means same value as previous row - * truncate trailing time from date-only values - * - * Consider disambiguating sql NULL and empty string. NULL is more common so using a shorter - * encoding for NULL and longer for empty string makes sense (e.g. "\u0000") - * - * TODO: binary/blob length prefixed encoding - * - * Note that while writeResults_text tries to not generate extra Strings to GC, I think the actual PrintWriter - * implementation is generating strings inside out.write(). So this is probably not much different from a GC - * perspective. - */ - void writeResults_compact(PrintWriter out, Results rs, String sep, String eol) throws SQLException - { - final int count = rs.getMetaData().getColumnCount(); - final boolean serializeDateAsNumber=false; + @Test + public void testExecuteCompact() throws Exception + { + MockHttpServletResponse response = executeSql("lists", + "SELECT Name, Age FROM " + LIST_NAME + " ORDER BY Age, Name", Format.compact); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String sep = String.valueOf(US); + + String eol = String.valueOf(RS); + String ditto = String.valueOf(BS); + + String[] records = content.split(eol); + // records: [0]=meta, [1]=column names, [2]=types, [3..5]=data rows + assertTrue("Expected at least 6 records", records.length >= 6); + + // Column names + String[] colNames = records[1].split(sep); + assertEquals("Name", colNames[0]); + assertEquals("Age", colNames[1]); + + // First data row: Alice, 30 + String[] row1 = records[3].split(sep, -1); + assertEquals("Alice", row1[0]); + assertEquals("30", row1[1]); + + // Second data row: Bob, 30 (ditto marker since Age repeats) + String[] row2 = records[4].split(sep, -1); + assertEquals("Bob", row2[0]); + assertEquals(ditto, row2[1]); + + // Third data row: Carol, 35 + String[] row3 = records[5].split(sep, -1); + assertEquals("Carol", row3[0]); + assertEquals("35", row3[1]); + } - // meta-meta-data - out.write("18.2"+sep+"name"+sep+"jdbcType"+eol); + @Test + public void testExecuteTsv() throws Exception + { + MockHttpServletResponse response = executeSql("lists", + "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", Format.tsv); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String[] lines = content.split("\n"); + assertTrue("Expected at least 4 lines (header + 3 data rows), got " + lines.length, lines.length >= 4); + + // Header row: column names + String[] headers = lines[0].split("\t"); + assertEquals("Name", headers[0]); + assertEquals("Age", headers[1]); + assertEquals("Score", headers[2]); + + // Data rows ordered by Name + String[] row1 = lines[1].split("\t"); + assertEquals("Alice", row1[0]); + assertEquals("30", row1[1]); + assertEquals("95.5", row1[2]); + + String[] row2 = lines[2].split("\t"); + assertEquals("Bob", row2[0]); + assertEquals("30", row2[1]); + assertEquals("87.3", row2[2]); + + String[] row3 = lines[3].split("\t"); + assertEquals("Carol", row3[0]); + assertEquals("35", row3[1]); + assertEquals("91.0", row3[2]); + } - for (int i = 1; i <= count; i++) + @Test + public void testExecuteDate() throws Exception { - out.write(rs.getColumn(i).getName()); - out.write(i == count ? eol : sep); + // Date columns should render as ISO-8601 (space instead of 'T') via DisplayColumn.getTsvFormattedValue() + MockHttpServletResponse response = executeSql("lists", + "SELECT Name, Created FROM " + LIST_NAME + " ORDER BY Name", Format.split); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + String[] tokens = response.getContentAsString().split("[\t\n]"); + // tokens: meta(3) + colNames(2) + jdbcTypes(2) + data(3 rows * 2 cols = 6) = 13 minimum + assertTrue("Expected at least 13 tokens, got " + tokens.length, tokens.length >= 13); + assertEquals("Alice", tokens[7]); + assertTrue("Created date should be ISO-8601: " + tokens[8], + tokens[8].matches("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.*")); } - // pull types from ResultSetMetaData, not ColumnInfo - JdbcType[] types = new JdbcType[count + 1]; - for (int i = 1; i <= count; i++) + @Test + public void testNoSql() throws Exception { - JdbcType jdbc = JdbcType.valueOf(rs.getMetaData().getColumnType(i)); - types[i] = jdbc; - out.write(jdbc.name()); - out.write(i == count ? eol : sep); + MockHttpServletResponse response = executeSql("lists", null, Format.split); + assertTrue("Expected error about missing SQL", + response.getContentAsString().contains("No sql provided")); } - String DITTO = "\u0008"; - String[] prev = new String[count+1]; - String[] row = new String[count+1]; + @Test + public void testSchemaNotFound() throws Exception + { + MockHttpServletResponse response = executeSql("nonexistent", + "SELECT 1", Format.tsv); + assertTrue("Expected schema not found error", + response.getContentAsString().contains("Schema not found")); + } - while (rs.next()) + @Test + public void testTruncation() throws Exception { - for (int column = 1; column <= count; column++) - { - switch (types[column]) - { - case TINYINT: - case SMALLINT: - case INTEGER: - case CHAR: - case VARCHAR: - case DOUBLE: - case REAL: - default: - { - row[column] = rs.getString(column); - break; - } - case TIMESTAMP: - { - Date date = rs.getTimestamp(column); - if (null == date) - row[column] = null; - else if (serializeDateAsNumber) - row[column] = Long.toString(date.getTime()); - else - { - String d = DateUtil.formatIsoDateShortTime(date); - if (d.endsWith(" 00:00")) - d = d.substring(0,d.length()-6); - row[column] = d; - } - break; - } - case BOOLEAN: - { - boolean b = rs.getBoolean(column); - row[column] = rs.wasNull() ? null : b ? "1": "0"; - break; - } - case DECIMAL: - { - BigDecimal dec = rs.getBigDecimal(column); - row[column] = null==dec ? null : dec.toPlainString(); - break; - } - } - } - for (int column = 1; column <= count; column++) - { - String s = row[column]; - if (null != s && !s.isEmpty()) - { - if (s.equals(prev[column])) - out.write(DITTO); - else - out.write(s); - } - out.write(column == count ? eol : sep); - } - String[] t = prev; - prev = row; - row = t; + User user = TestContext.get().getUser(); + UserSchema listsSchema = (UserSchema) DefaultSchema.get(user, _folder).getSchema("lists"); + ContainerUser cu = ContainerUser.create(_folder, user); + + // Truncate at 3 chars. Alice/Carol (5 chars) should be truncated; Bob (3 chars == limit, not >) should not. + var execute = new SqlExecute(cu, listsSchema, "SELECT Name FROM " + LIST_NAME + " ORDER BY Name") + .truncation(3, "…[truncated]"); + + StringWriter sw = new StringWriter(); + execute.execute(sw); + String[] lines = sw.toString().split("\n"); + + assertEquals("Name", lines[0]); + assertEquals("Ali…[truncated]", lines[1]); // 5 > 3: truncated + assertEquals("Bob", lines[2]); // 3 == 3: not truncated (boundary) + assertEquals("Car…[truncated]", lines[3]); // 5 > 3: truncated + } + + private SqlExecute.ExecuteResult executeWithLimit(int limit) throws Exception + { + User user = TestContext.get().getUser(); + UserSchema listsSchema = (UserSchema) DefaultSchema.get(user, _folder).getSchema("lists"); + ContainerUser cu = ContainerUser.create(_folder, user); + var execute = new SqlExecute(cu, listsSchema, "SELECT Name FROM " + LIST_NAME + " ORDER BY Name") + .page(0, limit); + return execute.execute(new StringWriter()); + } + + @Test + public void testLimit_lessThanRowCount() throws Exception + { + // 3 rows in table, limit=2 → incomplete result + var result = executeWithLimit(2); + assertEquals(2, result.rows()); + assertFalse("Expected more rows to be available", result.complete()); + } + + @Test + public void testLimit_equalToRowCount() throws Exception + { + // 3 rows in table, limit=3 → complete result + var result = executeWithLimit(3); + assertEquals(3, result.rows()); + assertTrue("Expected result to be complete", result.complete()); + } + + @Test + public void testLimit_greaterThanRowCount() throws Exception + { + // 3 rows in table, limit=10 → complete result + var result = executeWithLimit(10); + assertEquals(3, result.rows()); + assertTrue("Expected result to be complete", result.complete()); } - out.flush(); } -} \ No newline at end of file +} diff --git a/query/src/org/labkey/query/controllers/prompts/LabKeySql.md b/query/src/org/labkey/query/controllers/prompts/LabKeySql.md index 256d18fb1df..51666e4d111 100644 --- a/query/src/org/labkey/query/controllers/prompts/LabKeySql.md +++ b/query/src/org/labkey/query/controllers/prompts/LabKeySql.md @@ -34,6 +34,12 @@ LabKey SQL allows you to create virtual columns within a query by using SQL expr `SELECT expression AS column_name FROM table` * **Functionality:** The syntax involves performing a calculation and then aliasing the result with a new column name using the `as` keyword. +* **Reserved words as aliases:** SQL function names (`COUNT`, `SUM`, `AVG`, `MIN`, `MAX`) and other SQL keywords are reserved and cannot be used as bare column aliases. Quote them or choose a different name: + ```sql + -- Wrong: COUNT(MouseId) AS Count → parse error + -- Right: COUNT(MouseId) AS "Count" + -- Right: COUNT(MouseId) AS TotalCount + ``` * **Examples:** * **Pulse Pressure:** To calculate pulse pressure from systolic and diastolic blood pressure values: ```sql