diff --git a/tests/data/swe-bench/swe-bench-dev.json b/tests/data/swe-bench/swe-bench-dev.json deleted file mode 100644 index 2b9a0de81..000000000 --- a/tests/data/swe-bench/swe-bench-dev.json +++ /dev/null @@ -1 +0,0 @@ -[{"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-4764", "base_commit": "a820c139ccbe6d1865d73c4a459945cd69899f8f", "patch": "diff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py\n--- a/src/sqlfluff/cli/commands.py\n+++ b/src/sqlfluff/cli/commands.py\n@@ -44,6 +44,7 @@\n dialect_selector,\n dialect_readout,\n )\n+from sqlfluff.core.linter import LintingResult\n from sqlfluff.core.config import progress_bar_configuration\n \n from sqlfluff.core.enums import FormatType, Color\n@@ -691,12 +692,16 @@ def lint(\n sys.exit(EXIT_SUCCESS)\n \n \n-def do_fixes(lnt, result, formatter=None, **kwargs):\n+def do_fixes(\n+ result: LintingResult, formatter: Optional[OutputStreamFormatter] = None, **kwargs\n+):\n \"\"\"Actually do the fixes.\"\"\"\n- click.echo(\"Persisting Changes...\")\n+ if formatter and formatter.verbosity >= 0:\n+ click.echo(\"Persisting Changes...\")\n res = result.persist_changes(formatter=formatter, **kwargs)\n if all(res.values()):\n- click.echo(\"Done. Please check your files to confirm.\")\n+ if formatter and formatter.verbosity >= 0:\n+ click.echo(\"Done. Please check your files to confirm.\")\n return True\n # If some failed then return false\n click.echo(\n@@ -708,7 +713,7 @@ def do_fixes(lnt, result, formatter=None, **kwargs):\n return False # pragma: no cover\n \n \n-def _stdin_fix(linter, formatter, fix_even_unparsable):\n+def _stdin_fix(linter: Linter, formatter, fix_even_unparsable):\n \"\"\"Handle fixing from stdin.\"\"\"\n exit_code = EXIT_SUCCESS\n stdin = sys.stdin.read()\n@@ -751,7 +756,7 @@ def _stdin_fix(linter, formatter, fix_even_unparsable):\n \n \n def _paths_fix(\n- linter,\n+ linter: Linter,\n formatter,\n paths,\n processes,\n@@ -765,11 +770,12 @@ def _paths_fix(\n ):\n \"\"\"Handle fixing from paths.\"\"\"\n # Lint the paths (not with the fix argument at this stage), outputting as we go.\n- click.echo(\"==== finding fixable violations ====\")\n+ if formatter.verbosity >= 0:\n+ click.echo(\"==== finding fixable violations ====\")\n exit_code = EXIT_SUCCESS\n \n with PathAndUserErrorHandler(formatter):\n- result = linter.lint_paths(\n+ result: LintingResult = linter.lint_paths(\n paths,\n fix=True,\n ignore_non_existent_files=False,\n@@ -781,20 +787,18 @@ def _paths_fix(\n \n # NB: We filter to linting violations here, because they're\n # the only ones which can be potentially fixed.\n- if result.num_violations(types=SQLLintError, fixable=True) > 0:\n- click.echo(\"==== fixing violations ====\")\n- click.echo(\n- f\"{result.num_violations(types=SQLLintError, fixable=True)} fixable \"\n- \"linting violations found\"\n- )\n+ num_fixable = result.num_violations(types=SQLLintError, fixable=True)\n+ if num_fixable > 0:\n+ if formatter.verbosity >= 0:\n+ click.echo(\"==== fixing violations ====\")\n+ click.echo(f\"{num_fixable} \" \"fixable linting violations found\")\n if force:\n- if warn_force:\n+ if warn_force and formatter.verbosity >= 0:\n click.echo(\n f\"{formatter.colorize('FORCE MODE', Color.red)}: \"\n \"Attempting fixes...\"\n )\n success = do_fixes(\n- linter,\n result,\n formatter,\n types=SQLLintError,\n@@ -809,9 +813,9 @@ def _paths_fix(\n c = click.getchar().lower()\n click.echo(\"...\")\n if c in (\"y\", \"\\r\", \"\\n\"):\n- click.echo(\"Attempting fixes...\")\n+ if formatter.verbosity >= 0:\n+ click.echo(\"Attempting fixes...\")\n success = do_fixes(\n- linter,\n result,\n formatter,\n types=SQLLintError,\n@@ -829,8 +833,9 @@ def _paths_fix(\n click.echo(\"Aborting...\")\n exit_code = EXIT_FAIL\n else:\n- click.echo(\"==== no fixable linting violations found ====\")\n- formatter.completion_message()\n+ if formatter.verbosity >= 0:\n+ click.echo(\"==== no fixable linting violations found ====\")\n+ formatter.completion_message()\n \n error_types = [\n (\n@@ -841,7 +846,7 @@ def _paths_fix(\n ]\n for num_violations_kwargs, message_format, error_level in error_types:\n num_violations = result.num_violations(**num_violations_kwargs)\n- if num_violations > 0:\n+ if num_violations > 0 and formatter.verbosity >= 0:\n click.echo(message_format.format(num_violations))\n exit_code = max(exit_code, error_level)\n \n@@ -880,10 +885,20 @@ def _paths_fix(\n \"--force\",\n is_flag=True,\n help=(\n- \"skip the confirmation prompt and go straight to applying \"\n+ \"Skip the confirmation prompt and go straight to applying \"\n \"fixes. **Use this with caution.**\"\n ),\n )\n+@click.option(\n+ \"-q\",\n+ \"--quiet\",\n+ is_flag=True,\n+ help=(\n+ \"Reduces the amount of output to stdout to a minimal level. \"\n+ \"This is effectively the opposite of -v. NOTE: It will only \"\n+ \"take effect if -f/--force is also set.\"\n+ ),\n+)\n @click.option(\n \"-x\",\n \"--fixed-suffix\",\n@@ -913,6 +928,7 @@ def fix(\n force: bool,\n paths: Tuple[str],\n bench: bool = False,\n+ quiet: bool = False,\n fixed_suffix: str = \"\",\n logger: Optional[logging.Logger] = None,\n processes: Optional[int] = None,\n@@ -932,6 +948,13 @@ def fix(\n \"\"\"\n # some quick checks\n fixing_stdin = (\"-\",) == paths\n+ if quiet:\n+ if kwargs[\"verbose\"]:\n+ click.echo(\n+ \"ERROR: The --quiet flag can only be used if --verbose is not set.\",\n+ )\n+ sys.exit(EXIT_ERROR)\n+ kwargs[\"verbose\"] = -1\n \n config = get_config(\n extra_config_path, ignore_local_config, require_dialect=False, **kwargs\ndiff --git a/src/sqlfluff/cli/formatters.py b/src/sqlfluff/cli/formatters.py\n--- a/src/sqlfluff/cli/formatters.py\n+++ b/src/sqlfluff/cli/formatters.py\n@@ -94,7 +94,7 @@ def __init__(\n ):\n self._output_stream = output_stream\n self.plain_output = self.should_produce_plain_output(nocolor)\n- self._verbosity = verbosity\n+ self.verbosity = verbosity\n self._filter_empty = filter_empty\n self.output_line_length = output_line_length\n \n@@ -116,13 +116,13 @@ def _format_config(self, linter: Linter) -> str:\n \"\"\"Format the config of a `Linter`.\"\"\"\n text_buffer = StringIO()\n # Only show version information if verbosity is high enough\n- if self._verbosity > 0:\n+ if self.verbosity > 0:\n text_buffer.write(\"==== sqlfluff ====\\n\")\n config_content = [\n (\"sqlfluff\", get_package_version()),\n (\"python\", get_python_version()),\n (\"implementation\", get_python_implementation()),\n- (\"verbosity\", self._verbosity),\n+ (\"verbosity\", self.verbosity),\n ]\n if linter.dialect:\n config_content.append((\"dialect\", linter.dialect.name))\n@@ -138,7 +138,7 @@ def _format_config(self, linter: Linter) -> str:\n col_width=41,\n )\n )\n- if self._verbosity > 1:\n+ if self.verbosity > 1:\n text_buffer.write(\"\\n== Raw Config:\\n\")\n text_buffer.write(self.format_config_vals(linter.config.iter_vals()))\n return text_buffer.getvalue()\n@@ -150,7 +150,7 @@ def dispatch_config(self, linter: Linter) -> None:\n def dispatch_persist_filename(self, filename, result):\n \"\"\"Dispatch filenames during a persist operation.\"\"\"\n # Only show the skip records at higher levels of verbosity\n- if self._verbosity >= 2 or result != \"SKIP\":\n+ if self.verbosity >= 2 or result != \"SKIP\":\n self._dispatch(self.format_filename(filename=filename, success=result))\n \n def _format_path(self, path: str) -> str:\n@@ -159,14 +159,14 @@ def _format_path(self, path: str) -> str:\n \n def dispatch_path(self, path: str) -> None:\n \"\"\"Dispatch paths for display.\"\"\"\n- if self._verbosity > 0:\n+ if self.verbosity > 0:\n self._dispatch(self._format_path(path))\n \n def dispatch_template_header(\n self, fname: str, linter_config: FluffConfig, file_config: FluffConfig\n ) -> None:\n \"\"\"Dispatch the header displayed before templating.\"\"\"\n- if self._verbosity > 1:\n+ if self.verbosity > 1:\n self._dispatch(self.format_filename(filename=fname, success=\"TEMPLATING\"))\n # This is where we output config diffs if they exist.\n if file_config:\n@@ -182,12 +182,12 @@ def dispatch_template_header(\n \n def dispatch_parse_header(self, fname: str) -> None:\n \"\"\"Dispatch the header displayed before parsing.\"\"\"\n- if self._verbosity > 1:\n+ if self.verbosity > 1:\n self._dispatch(self.format_filename(filename=fname, success=\"PARSING\"))\n \n def dispatch_lint_header(self, fname: str, rules: List[str]) -> None:\n \"\"\"Dispatch the header displayed before linting.\"\"\"\n- if self._verbosity > 1:\n+ if self.verbosity > 1:\n self._dispatch(\n self.format_filename(\n filename=fname, success=f\"LINTING ({', '.join(rules)})\"\n@@ -202,7 +202,7 @@ def dispatch_compilation_header(self, templater, message):\n \n def dispatch_processing_header(self, processes: int) -> None:\n \"\"\"Dispatch the header displayed before linting.\"\"\"\n- if self._verbosity > 0:\n+ if self.verbosity > 0:\n self._dispatch( # pragma: no cover\n f\"{self.colorize('effective configured processes: ', Color.lightgrey)} \"\n f\"{processes}\"\n@@ -228,7 +228,7 @@ def _format_file_violations(\n show = fails + warns > 0\n \n # Only print the filename if it's either a failure or verbosity > 1\n- if self._verbosity > 0 or show:\n+ if self.verbosity > 0 or show:\n text_buffer.write(self.format_filename(fname, success=fails == 0))\n text_buffer.write(\"\\n\")\n \n@@ -253,6 +253,8 @@ def dispatch_file_violations(\n self, fname: str, linted_file: LintedFile, only_fixable: bool\n ) -> None:\n \"\"\"Dispatch any violations found in a file.\"\"\"\n+ if self.verbosity < 0:\n+ return\n s = self._format_file_violations(\n fname,\n linted_file.get_violations(\n@@ -392,10 +394,13 @@ def format_filename(\n if isinstance(success, str):\n status_string = success\n else:\n- status_string = self.colorize(\n- success_text if success else \"FAIL\",\n- Color.green if success else Color.red,\n- )\n+ status_string = success_text if success else \"FAIL\"\n+\n+ if status_string in (\"PASS\", \"FIXED\", success_text):\n+ status_string = self.colorize(status_string, Color.green)\n+ elif status_string in (\"FAIL\", \"ERROR\"):\n+ status_string = self.colorize(status_string, Color.red)\n+\n return f\"== [{self.colorize(filename, Color.lightgrey)}] {status_string}\"\n \n def format_violation(\ndiff --git a/src/sqlfluff/core/linter/linted_dir.py b/src/sqlfluff/core/linter/linted_dir.py\n--- a/src/sqlfluff/core/linter/linted_dir.py\n+++ b/src/sqlfluff/core/linter/linted_dir.py\n@@ -117,7 +117,11 @@ def persist_changes(\n for file in self.files:\n if file.num_violations(fixable=True, **kwargs) > 0:\n buffer[file.path] = file.persist_tree(suffix=fixed_file_suffix)\n- result = buffer[file.path]\n+ result: Union[bool, str]\n+ if buffer[file.path] is True:\n+ result = \"FIXED\"\n+ else: # pragma: no cover\n+ result = buffer[file.path]\n else: # pragma: no cover TODO?\n buffer[file.path] = True\n result = \"SKIP\"\n", "test_patch": "diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py\n--- a/test/cli/commands_test.py\n+++ b/test/cli/commands_test.py\n@@ -557,6 +557,18 @@ def test__cli__command_lint_parse(command):\n ),\n 1,\n ),\n+ # Test that setting --quiet with --verbose raises an error.\n+ (\n+ (\n+ fix,\n+ [\n+ \"--quiet\",\n+ \"--verbose\",\n+ \"test/fixtures/cli/fail_many.sql\",\n+ ],\n+ ),\n+ 2,\n+ ),\n ],\n )\n def test__cli__command_lint_parse_with_retcode(command, ret_code):\n@@ -1891,7 +1903,7 @@ def test_cli_fix_disabled_progress_bar_deprecated_option(\n \n \n def test__cli__fix_multiple_errors_no_show_errors():\n- \"\"\"Basic checking of lint functionality.\"\"\"\n+ \"\"\"Test the fix output.\"\"\"\n result = invoke_assert_code(\n ret_code=1,\n args=[\n@@ -1910,8 +1922,57 @@ def test__cli__fix_multiple_errors_no_show_errors():\n assert result.output.replace(\"\\\\\", \"/\").startswith(multiple_expected_output)\n \n \n+def test__cli__fix_multiple_errors_quiet_force():\n+ \"\"\"Test the fix --quiet option with --force.\"\"\"\n+ result = invoke_assert_code(\n+ ret_code=0,\n+ args=[\n+ fix,\n+ [\n+ \"--disable-progress-bar\",\n+ \"test/fixtures/linter/multiple_sql_errors.sql\",\n+ \"--force\",\n+ \"--quiet\",\n+ \"-x\",\n+ \"_fix\",\n+ ],\n+ ],\n+ )\n+ normalised_output = result.output.replace(\"\\\\\", \"/\")\n+ assert normalised_output.startswith(\n+ \"\"\"1 fixable linting violations found\n+== [test/fixtures/linter/multiple_sql_errors.sql] FIXED\"\"\"\n+ )\n+\n+\n+def test__cli__fix_multiple_errors_quiet_no_force():\n+ \"\"\"Test the fix --quiet option without --force.\"\"\"\n+ result = invoke_assert_code(\n+ ret_code=0,\n+ args=[\n+ fix,\n+ [\n+ \"--disable-progress-bar\",\n+ \"test/fixtures/linter/multiple_sql_errors.sql\",\n+ \"--quiet\",\n+ \"-x\",\n+ \"_fix\",\n+ ],\n+ # Test with the confirmation step.\n+ \"y\",\n+ ],\n+ )\n+ normalised_output = result.output.replace(\"\\\\\", \"/\")\n+ assert normalised_output.startswith(\n+ \"\"\"1 fixable linting violations found\n+Are you sure you wish to attempt to fix these? [Y/n] ...\n+== [test/fixtures/linter/multiple_sql_errors.sql] FIXED\n+All Finished\"\"\"\n+ )\n+\n+\n def test__cli__fix_multiple_errors_show_errors():\n- \"\"\"Basic checking of lint functionality.\"\"\"\n+ \"\"\"Test the fix --show-lint-violations option.\"\"\"\n result = invoke_assert_code(\n ret_code=1,\n args=[\n", "problem_statement": "Enable quiet mode/no-verbose in CLI for use in pre-commit hook\nThere seems to be only an option to increase the level of verbosity when using SQLFluff [CLI](https://docs.sqlfluff.com/en/stable/cli.html), not to limit it further.\r\n\r\nIt would be great to have an option to further limit the amount of prints when running `sqlfluff fix`, especially in combination with deployment using a pre-commit hook. For example, only print the return status and the number of fixes applied, similar to how it is when using `black` in a pre-commit hook:\r\n![image](https://user-images.githubusercontent.com/10177212/140480676-dc98d00b-4383-44f2-bb90-3301a6eedec2.png)\r\n\r\nThis hides the potentially long list of fixes that are being applied to the SQL files, which can get quite verbose.\n", "hints_text": "", "created_at": "2023-04-16T14:24:42Z", "version": "1.4", "FAIL_TO_PASS": ["test/cli/commands_test.py::test__cli__fix_multiple_errors_quiet_force", "test/cli/commands_test.py::test__cli__fix_multiple_errors_quiet_no_force"], "PASS_TO_PASS": ["test/cli/commands_test.py::test__cli__command_directed", "test/cli/commands_test.py::test__cli__command_dialect", "test/cli/commands_test.py::test__cli__command_no_dialect", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_explicit_warning", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_implicit_warning", "test/cli/commands_test.py::test__cli__command_dialect_legacy", "test/cli/commands_test.py::test__cli__command_extra_config_fail", "test/cli/commands_test.py::test__cli__command_lint_stdin[command0]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command1]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command2]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command3]", "test/cli/commands_test.py::test__cli__command_render_stdin", "test/cli/commands_test.py::test__cli__command_lint_parse[command0]", "test/cli/commands_test.py::test__cli__command_lint_parse[command1]", "test/cli/commands_test.py::test__cli__command_lint_parse[command2]", "test/cli/commands_test.py::test__cli__command_lint_parse[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command4]", "test/cli/commands_test.py::test__cli__command_lint_parse[command5]", "test/cli/commands_test.py::test__cli__command_lint_parse[command6]", "test/cli/commands_test.py::test__cli__command_lint_parse[command7]", "test/cli/commands_test.py::test__cli__command_lint_parse[command8]", "test/cli/commands_test.py::test__cli__command_lint_parse[command9]", "test/cli/commands_test.py::test__cli__command_lint_parse[command10]", "test/cli/commands_test.py::test__cli__command_lint_parse[command11]", "test/cli/commands_test.py::test__cli__command_lint_parse[command12]", "test/cli/commands_test.py::test__cli__command_lint_parse[command13]", "test/cli/commands_test.py::test__cli__command_lint_parse[command14]", "test/cli/commands_test.py::test__cli__command_lint_parse[command15]", "test/cli/commands_test.py::test__cli__command_lint_parse[command16]", "test/cli/commands_test.py::test__cli__command_lint_parse[command17]", "test/cli/commands_test.py::test__cli__command_lint_parse[command18]", "test/cli/commands_test.py::test__cli__command_lint_parse[command19]", "test/cli/commands_test.py::test__cli__command_lint_parse[command20]", "test/cli/commands_test.py::test__cli__command_lint_parse[command21]", "test/cli/commands_test.py::test__cli__command_lint_parse[command22]", "test/cli/commands_test.py::test__cli__command_lint_parse[command23]", "test/cli/commands_test.py::test__cli__command_lint_parse[command24]", "test/cli/commands_test.py::test__cli__command_lint_parse[command25]", "test/cli/commands_test.py::test__cli__command_lint_parse[command26]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command0-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command1-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command2-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command3-0]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command4-0]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command5-2]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command6-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command7-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command8-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command9-2]", "test/cli/commands_test.py::test__cli__command_lint_warning_explicit_file_ignored", "test/cli/commands_test.py::test__cli__command_lint_skip_ignore_files", "test/cli/commands_test.py::test__cli__command_lint_ignore_local_config", "test/cli/commands_test.py::test__cli__command_lint_warning", "test/cli/commands_test.py::test__cli__command_versioning", "test/cli/commands_test.py::test__cli__command_version", "test/cli/commands_test.py::test__cli__command_rules", "test/cli/commands_test.py::test__cli__command_dialects", "test/cli/commands_test.py::test__cli__command__fix[LT01-test/fixtures/linter/indentation_errors.sql0]", "test/cli/commands_test.py::test__cli__command__fix[LT01-test/fixtures/linter/whitespace_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[LT01-test/fixtures/linter/indentation_errors.sql1]", "test/cli/commands_test.py::test__cli__command__fix[LT02-test/fixtures/linter/indentation_error_hard.sql]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_templating_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error_FIX_EVEN_UNPARSABLE]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[2_files_with_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-True]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-True]", "test/cli/commands_test.py::test__cli__fix_loop_limit_behavior[--", "test/cli/commands_test.py::test__cli__command_fix_stdin[select", "test/cli/commands_test.py::test__cli__command_fix_stdin[", "test/cli/commands_test.py::test__cli__command_format_stdin[select", "test/cli/commands_test.py::test__cli__command_format_stdin[", "test/cli/commands_test.py::test__cli__command_fix_stdin_logging_to_stderr", "test/cli/commands_test.py::test__cli__command_fix_stdin_safety", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[create", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[select", "test/cli/commands_test.py::test__cli__command__fix_no_force[LT01-test/fixtures/linter/indentation_errors.sql-y-0-0]", "test/cli/commands_test.py::test__cli__command__fix_no_force[LT01-test/fixtures/linter/indentation_errors.sql-n-1-1]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-json]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[select", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[SElect", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command0]", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command1]", "test/cli/commands_test.py::test__cli__command_lint_nocolor", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-none]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-none]", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation_native", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation-native]", "test/cli/commands_test.py::test___main___help", "test/cli/commands_test.py::test_encoding[utf-8-ascii]", "test/cli/commands_test.py::test_encoding[utf-8-sig-UTF-8-SIG]", "test/cli/commands_test.py::test_encoding[utf-32-UTF-32]", "test/cli/commands_test.py::test_cli_encoding[utf-8-command-line-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-command-line-True]", "test/cli/commands_test.py::test_cli_encoding[utf-8-config-file-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-config-file-True]", "test/cli/commands_test.py::test_cli_no_disable_noqa_flag", "test/cli/commands_test.py::test_cli_disable_noqa_flag", "test/cli/commands_test.py::test_cli_get_default_config", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar_deprecated_option", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_paths", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_files", "test/cli/commands_test.py::TestProgressBars::test_cli_fix_disabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_fix_disabled_progress_bar_deprecated_option", "test/cli/commands_test.py::test__cli__fix_multiple_errors_no_show_errors", "test/cli/commands_test.py::test__cli__fix_multiple_errors_show_errors", "test/cli/commands_test.py::test__cli__multiple_files__fix_multiple_errors_show_errors", "test/cli/commands_test.py::test__cli__render_fail", "test/cli/commands_test.py::test__cli__render_pass"], "environment_setup_commit": "d19de0ecd16d298f9e3bfb91da122734c40c01e5"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-2862", "base_commit": "447ecf862a4d2b977d0add9f444655357b9c4f1f", "patch": "diff --git a/src/sqlfluff/core/linter/common.py b/src/sqlfluff/core/linter/common.py\n--- a/src/sqlfluff/core/linter/common.py\n+++ b/src/sqlfluff/core/linter/common.py\n@@ -67,21 +67,3 @@ class ParsedString(NamedTuple):\n config: FluffConfig\n fname: str\n source_str: str\n-\n-\n-class EnrichedFixPatch(NamedTuple):\n- \"\"\"An edit patch for a source file.\"\"\"\n-\n- source_slice: slice\n- templated_slice: slice\n- fixed_raw: str\n- # The patch category, functions mostly for debugging and explanation\n- # than for function. It allows traceability of *why* this patch was\n- # generated.\n- patch_category: str\n- templated_str: str\n- source_str: str\n-\n- def dedupe_tuple(self):\n- \"\"\"Generate a tuple of this fix for deduping.\"\"\"\n- return (self.source_slice, self.fixed_raw)\ndiff --git a/src/sqlfluff/core/linter/linted_file.py b/src/sqlfluff/core/linter/linted_file.py\n--- a/src/sqlfluff/core/linter/linted_file.py\n+++ b/src/sqlfluff/core/linter/linted_file.py\n@@ -30,9 +30,9 @@\n from sqlfluff.core.templaters import TemplatedFile\n \n # Classes needed only for type checking\n-from sqlfluff.core.parser.segments.base import BaseSegment, FixPatch\n+from sqlfluff.core.parser.segments.base import BaseSegment, FixPatch, EnrichedFixPatch\n \n-from sqlfluff.core.linter.common import NoQaDirective, EnrichedFixPatch\n+from sqlfluff.core.linter.common import NoQaDirective\n \n # Instantiate the linter logger\n linter_logger: logging.Logger = logging.getLogger(\"sqlfluff.linter\")\n@@ -203,9 +203,7 @@ def is_clean(self) -> bool:\n return not any(self.get_violations(filter_ignore=True))\n \n @staticmethod\n- def _log_hints(\n- patch: Union[EnrichedFixPatch, FixPatch], templated_file: TemplatedFile\n- ):\n+ def _log_hints(patch: FixPatch, templated_file: TemplatedFile):\n \"\"\"Log hints for debugging during patch generation.\"\"\"\n # This next bit is ALL FOR LOGGING AND DEBUGGING\n max_log_length = 10\n@@ -279,18 +277,16 @@ def fix_string(self) -> Tuple[Any, bool]:\n dedupe_buffer = []\n # We use enumerate so that we get an index for each patch. This is entirely\n # so when debugging logs we can find a given patch again!\n- patch: Union[EnrichedFixPatch, FixPatch]\n+ patch: FixPatch # Could be FixPatch or its subclass, EnrichedFixPatch\n for idx, patch in enumerate(\n- self.tree.iter_patches(templated_str=self.templated_file.templated_str)\n+ self.tree.iter_patches(templated_file=self.templated_file)\n ):\n linter_logger.debug(\" %s Yielded patch: %s\", idx, patch)\n self._log_hints(patch, self.templated_file)\n \n- # Attempt to convert to source space.\n+ # Get source_slice.\n try:\n- source_slice = self.templated_file.templated_slice_to_source_slice(\n- patch.templated_slice,\n- )\n+ enriched_patch = patch.enrich(self.templated_file)\n except ValueError: # pragma: no cover\n linter_logger.info(\n \" - Skipping. Source space Value Error. i.e. attempted \"\n@@ -301,10 +297,10 @@ def fix_string(self) -> Tuple[Any, bool]:\n continue\n \n # Check for duplicates\n- dedupe_tuple = (source_slice, patch.fixed_raw)\n- if dedupe_tuple in dedupe_buffer:\n+ if enriched_patch.dedupe_tuple() in dedupe_buffer:\n linter_logger.info(\n- \" - Skipping. Source space Duplicate: %s\", dedupe_tuple\n+ \" - Skipping. Source space Duplicate: %s\",\n+ enriched_patch.dedupe_tuple(),\n )\n continue\n \n@@ -318,19 +314,10 @@ def fix_string(self) -> Tuple[Any, bool]:\n \n # Get the affected raw slices.\n local_raw_slices = self.templated_file.raw_slices_spanning_source_slice(\n- source_slice\n+ enriched_patch.source_slice\n )\n local_type_list = [slc.slice_type for slc in local_raw_slices]\n \n- enriched_patch = EnrichedFixPatch(\n- source_slice=source_slice,\n- templated_slice=patch.templated_slice,\n- patch_category=patch.patch_category,\n- fixed_raw=patch.fixed_raw,\n- templated_str=self.templated_file.templated_str[patch.templated_slice],\n- source_str=self.templated_file.source_str[source_slice],\n- )\n-\n # Deal with the easy cases of 1) New code at end 2) only literals\n if not local_type_list or set(local_type_list) == {\"literal\"}:\n linter_logger.info(\ndiff --git a/src/sqlfluff/core/parser/lexer.py b/src/sqlfluff/core/parser/lexer.py\n--- a/src/sqlfluff/core/parser/lexer.py\n+++ b/src/sqlfluff/core/parser/lexer.py\n@@ -535,6 +535,31 @@ def elements_to_segments(\n )\n )\n \n+ # Generate placeholders for any source-only slices that *follow*\n+ # the last element. This happens, for example, if a Jinja templated\n+ # file ends with \"{% endif %}\", and there's no trailing newline.\n+ if idx == len(elements) - 1:\n+ so_slices = [\n+ so\n+ for so in source_only_slices\n+ if so.source_idx >= source_slice.stop\n+ ]\n+ for so_slice in so_slices:\n+ segment_buffer.append(\n+ TemplateSegment(\n+ pos_marker=PositionMarker(\n+ slice(so_slice.source_idx, so_slice.end_source_idx()),\n+ slice(\n+ element.template_slice.stop,\n+ element.template_slice.stop,\n+ ),\n+ templated_file,\n+ ),\n+ source_str=so_slice.raw,\n+ block_type=so_slice.slice_type,\n+ )\n+ )\n+\n # Convert to tuple before return\n return tuple(segment_buffer)\n \ndiff --git a/src/sqlfluff/core/parser/segments/base.py b/src/sqlfluff/core/parser/segments/base.py\n--- a/src/sqlfluff/core/parser/segments/base.py\n+++ b/src/sqlfluff/core/parser/segments/base.py\n@@ -13,7 +13,16 @@\n from copy import deepcopy\n from dataclasses import dataclass, field, replace\n from io import StringIO\n-from typing import Any, Callable, Dict, Optional, List, Tuple, NamedTuple, Iterator\n+from typing import (\n+ Any,\n+ Callable,\n+ Dict,\n+ Optional,\n+ List,\n+ Tuple,\n+ Iterator,\n+ Union,\n+)\n import logging\n \n from tqdm import tqdm\n@@ -36,21 +45,54 @@\n from sqlfluff.core.parser.matchable import Matchable\n from sqlfluff.core.parser.markers import PositionMarker\n from sqlfluff.core.parser.context import ParseContext\n+from sqlfluff.core.templaters.base import TemplatedFile\n \n # Instantiate the linter logger (only for use in methods involved with fixing.)\n linter_logger = logging.getLogger(\"sqlfluff.linter\")\n \n \n-class FixPatch(NamedTuple):\n+@dataclass\n+class FixPatch:\n \"\"\"An edit patch for a templated file.\"\"\"\n \n templated_slice: slice\n fixed_raw: str\n # The patch category, functions mostly for debugging and explanation\n # than for function. It allows traceability of *why* this patch was\n- # generated. It has no siginificance for processing.\n+ # generated. It has no significance for processing.\n patch_category: str\n \n+ def enrich(self, templated_file: TemplatedFile) -> \"EnrichedFixPatch\":\n+ \"\"\"Convert patch to source space.\"\"\"\n+ source_slice = templated_file.templated_slice_to_source_slice(\n+ self.templated_slice,\n+ )\n+ return EnrichedFixPatch(\n+ source_slice=source_slice,\n+ templated_slice=self.templated_slice,\n+ patch_category=self.patch_category,\n+ fixed_raw=self.fixed_raw,\n+ templated_str=templated_file.templated_str[self.templated_slice],\n+ source_str=templated_file.source_str[source_slice],\n+ )\n+\n+\n+@dataclass\n+class EnrichedFixPatch(FixPatch):\n+ \"\"\"An edit patch for a source file.\"\"\"\n+\n+ source_slice: slice\n+ templated_str: str\n+ source_str: str\n+\n+ def enrich(self, templated_file: TemplatedFile) -> \"EnrichedFixPatch\":\n+ \"\"\"No-op override of base class function.\"\"\"\n+ return self\n+\n+ def dedupe_tuple(self):\n+ \"\"\"Generate a tuple of this fix for deduping.\"\"\"\n+ return (self.source_slice, self.fixed_raw)\n+\n \n @dataclass\n class AnchorEditInfo:\n@@ -1176,7 +1218,9 @@ def _validate_segment_after_fixes(self, rule_code, dialect, fixes_applied, segme\n def _log_apply_fixes_check_issue(message, *args): # pragma: no cover\n linter_logger.critical(message, *args)\n \n- def iter_patches(self, templated_str: str) -> Iterator[FixPatch]:\n+ def iter_patches(\n+ self, templated_file: TemplatedFile\n+ ) -> Iterator[Union[EnrichedFixPatch, FixPatch]]:\n \"\"\"Iterate through the segments generating fix patches.\n \n The patches are generated in TEMPLATED space. This is important\n@@ -1188,6 +1232,7 @@ def iter_patches(self, templated_str: str) -> Iterator[FixPatch]:\n \"\"\"\n # Does it match? If so we can ignore it.\n assert self.pos_marker\n+ templated_str = templated_file.templated_str\n matches = self.raw == templated_str[self.pos_marker.templated_slice]\n if matches:\n return\n@@ -1256,7 +1301,7 @@ def iter_patches(self, templated_str: str) -> Iterator[FixPatch]:\n insert_buff = \"\"\n \n # Now we deal with any changes *within* the segment itself.\n- yield from segment.iter_patches(templated_str=templated_str)\n+ yield from segment.iter_patches(templated_file=templated_file)\n \n # Once we've dealt with any patches from the segment, update\n # our position markers.\n@@ -1266,13 +1311,22 @@ def iter_patches(self, templated_str: str) -> Iterator[FixPatch]:\n # or insert. Also valid if we still have an insertion buffer here.\n end_diff = self.pos_marker.templated_slice.stop - templated_idx\n if end_diff or insert_buff:\n- yield FixPatch(\n- slice(\n- self.pos_marker.templated_slice.stop - end_diff,\n- self.pos_marker.templated_slice.stop,\n- ),\n- insert_buff,\n+ source_slice = segment.pos_marker.source_slice\n+ templated_slice = slice(\n+ self.pos_marker.templated_slice.stop - end_diff,\n+ self.pos_marker.templated_slice.stop,\n+ )\n+ # By returning an EnrichedFixPatch (rather than FixPatch), which\n+ # includes a source_slice field, we ensure that fixes adjacent\n+ # to source-only slices (e.g. {% endif %}) are placed\n+ # appropriately relative to source-only slices.\n+ yield EnrichedFixPatch(\n+ source_slice=source_slice,\n+ templated_slice=templated_slice,\n patch_category=\"end_point\",\n+ fixed_raw=insert_buff,\n+ templated_str=templated_file.templated_str[templated_slice],\n+ source_str=templated_file.source_str[source_slice],\n )\n \n def edit(self, raw):\ndiff --git a/src/sqlfluff/core/rules/base.py b/src/sqlfluff/core/rules/base.py\n--- a/src/sqlfluff/core/rules/base.py\n+++ b/src/sqlfluff/core/rules/base.py\n@@ -656,16 +656,18 @@ def indent(self) -> str:\n space = \" \"\n return space * self.tab_space_size if self.indent_unit == \"space\" else tab\n \n- def is_final_segment(self, context: RuleContext) -> bool:\n+ def is_final_segment(self, context: RuleContext, filter_meta: bool = True) -> bool:\n \"\"\"Is the current segment the final segment in the parse tree.\"\"\"\n- if len(self.filter_meta(context.siblings_post)) > 0:\n+ siblings_post = context.siblings_post\n+ if filter_meta:\n+ siblings_post = self.filter_meta(siblings_post)\n+ if len(siblings_post) > 0:\n # This can only fail on the last segment\n return False\n elif len(context.segment.segments) > 0:\n # This can only fail on the last base segment\n return False\n- elif context.segment.is_meta:\n- # We can't fail on a meta segment\n+ elif filter_meta and context.segment.is_meta:\n return False\n else:\n # We know we are at a leaf of the tree but not necessarily at the end of the\n@@ -674,9 +676,9 @@ def is_final_segment(self, context: RuleContext) -> bool:\n # one.\n child_segment = context.segment\n for parent_segment in context.parent_stack[::-1]:\n- possible_children = [\n- s for s in parent_segment.segments if not s.is_meta\n- ]\n+ possible_children = parent_segment.segments\n+ if filter_meta:\n+ possible_children = [s for s in possible_children if not s.is_meta]\n if len(possible_children) > possible_children.index(child_segment) + 1:\n return False\n child_segment = parent_segment\ndiff --git a/src/sqlfluff/core/templaters/slicers/tracer.py b/src/sqlfluff/core/templaters/slicers/tracer.py\n--- a/src/sqlfluff/core/templaters/slicers/tracer.py\n+++ b/src/sqlfluff/core/templaters/slicers/tracer.py\n@@ -289,7 +289,6 @@ def _slice_template(self) -> List[RawFileSlice]:\n # parts of the tag at a time.\n unique_alternate_id = None\n alternate_code = None\n- trimmed_content = \"\"\n if elem_type.endswith(\"_end\") or elem_type == \"raw_begin\":\n block_type = block_types[elem_type]\n block_subtype = None\n@@ -436,6 +435,16 @@ def _slice_template(self) -> List[RawFileSlice]:\n \"endfor\",\n \"endif\",\n ):\n+ # Replace RawSliceInfo for this slice with one that has\n+ # alternate ID and code for tracking. This ensures, for\n+ # instance, that if a file ends with \"{% endif %} (with\n+ # no newline following), that we still generate a\n+ # TemplateSliceInfo for it.\n+ unique_alternate_id = self.next_slice_id()\n+ alternate_code = f\"{result[-1].raw}\\0{unique_alternate_id}_0\"\n+ self.raw_slice_info[result[-1]] = RawSliceInfo(\n+ unique_alternate_id, alternate_code, []\n+ )\n # Record potential forward jump over this block.\n self.raw_slice_info[result[stack[-1]]].next_slice_indices.append(\n block_idx\ndiff --git a/src/sqlfluff/rules/L009.py b/src/sqlfluff/rules/L009.py\n--- a/src/sqlfluff/rules/L009.py\n+++ b/src/sqlfluff/rules/L009.py\n@@ -91,7 +91,7 @@ def _eval(self, context: RuleContext) -> Optional[LintResult]:\n \n \"\"\"\n # We only care about the final segment of the parse tree.\n- if not self.is_final_segment(context):\n+ if not self.is_final_segment(context, filter_meta=False):\n return None\n \n # Include current segment for complete stack and reverse.\n", "test_patch": "diff --git a/test/api/simple_test.py b/test/api/simple_test.py\n--- a/test/api/simple_test.py\n+++ b/test/api/simple_test.py\n@@ -72,16 +72,16 @@\n \"description\": \"Keywords must be consistently upper case.\",\n },\n {\n- \"code\": \"L009\",\n+ \"code\": \"L014\",\n \"line_no\": 1,\n \"line_pos\": 34,\n- \"description\": \"Files must end with a single trailing newline.\",\n+ \"description\": \"Unquoted identifiers must be consistently lower case.\",\n },\n {\n- \"code\": \"L014\",\n+ \"code\": \"L009\",\n \"line_no\": 1,\n- \"line_pos\": 34,\n- \"description\": \"Unquoted identifiers must be consistently lower case.\",\n+ \"line_pos\": 41,\n+ \"description\": \"Files must end with a single trailing newline.\",\n },\n ]\n \ndiff --git a/test/core/templaters/jinja_test.py b/test/core/templaters/jinja_test.py\n--- a/test/core/templaters/jinja_test.py\n+++ b/test/core/templaters/jinja_test.py\n@@ -822,6 +822,10 @@ def test__templater_jinja_slice_template(test, result):\n (\"block_end\", slice(113, 127, None), slice(11, 11, None)),\n (\"block_start\", slice(27, 46, None), slice(11, 11, None)),\n (\"literal\", slice(46, 57, None), slice(11, 22, None)),\n+ (\"block_end\", slice(57, 70, None), slice(22, 22, None)),\n+ (\"block_start\", slice(70, 89, None), slice(22, 22, None)),\n+ (\"block_end\", slice(100, 113, None), slice(22, 22, None)),\n+ (\"block_end\", slice(113, 127, None), slice(22, 22, None)),\n ],\n ),\n (\n@@ -910,8 +914,20 @@ def test__templater_jinja_slice_template(test, result):\n (\"literal\", slice(91, 92, None), slice(0, 0, None)),\n (\"block_end\", slice(92, 104, None), slice(0, 0, None)),\n (\"literal\", slice(104, 113, None), slice(0, 9, None)),\n- (\"templated\", slice(113, 139, None), slice(9, 29, None)),\n- (\"literal\", slice(139, 156, None), slice(29, 46, None)),\n+ (\"templated\", slice(113, 139, None), slice(9, 28, None)),\n+ (\"literal\", slice(139, 156, None), slice(28, 28, None)),\n+ ],\n+ ),\n+ (\n+ # Test for issue 2822: Handle slicing when there's no newline after\n+ # the Jinja block end.\n+ \"{% if true %}\\nSELECT 1 + 1\\n{%- endif %}\",\n+ None,\n+ [\n+ (\"block_start\", slice(0, 13, None), slice(0, 0, None)),\n+ (\"literal\", slice(13, 26, None), slice(0, 13, None)),\n+ (\"literal\", slice(26, 27, None), slice(13, 13, None)),\n+ (\"block_end\", slice(27, 39, None), slice(13, 13, None)),\n ],\n ),\n ],\ndiff --git a/test/fixtures/rules/std_rule_cases/L009.yml b/test/fixtures/rules/std_rule_cases/L009.yml\n--- a/test/fixtures/rules/std_rule_cases/L009.yml\n+++ b/test/fixtures/rules/std_rule_cases/L009.yml\n@@ -33,3 +33,9 @@ test_pass_templated_macro_newlines:\n {{ columns }}\n {% endmacro %}\n SELECT {{ get_keyed_nulls(\"other_id\") }}\n+\n+test_fail_templated_no_newline:\n+ # Tricky because there's no newline at the end of the file (following the\n+ # templated code).\n+ fail_str: \"{% if true %}\\nSELECT 1 + 1\\n{%- endif %}\"\n+ fix_str: \"{% if true %}\\nSELECT 1 + 1\\n{%- endif %}\\n\"\n", "problem_statement": "fix keep adding new line on wrong place \n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nTo replicate this issue you can create a file eg. test.template.sql \r\n\r\n```\r\n{% if true %}\r\nSELECT 1 + 1\r\n{%- endif %}\r\n```\r\n\r\nthen run:\r\n```\r\nsqlfluff fix test.template.sql \r\n```\r\n\r\nThis will give you:\r\n```\r\nL: 2 | P: 12 | L009 | Files must end with a trailing newline.\r\n```\r\n\r\nAnd the result of the file is now:\r\n```\r\n{% if true %}\r\nSELECT 1 + 1\r\n\r\n{%- endif %}\r\n```\r\n\r\nIf i run it again it will complain on the same issue and the result of the file would be: \r\n```\r\n{% if true %}\r\nSELECT 1 + 1\r\n\r\n\r\n{%- endif %}\r\n```\r\n\r\nAnd so on. \n\n### Expected Behaviour\n\nThe expected behavior would be to add the new line at the end of the file, that is after `{%- endif %}` instead of adding the new line at the end of the SQL query - so the result should look like this: \r\n\r\n```\r\n{% if true %}\r\nSELECT 1 + 1\r\n{%- endif %}\r\n\r\n```\n\n### Observed Behaviour\n\nAdds a new line to the end of the SQL query instead of in the end of the file. \n\n### How to reproduce\n\nAlready mentioned above (in What Happened section).\n\n### Dialect\n\nsnowflake\n\n### Version\n\nsqlfluff, version 0.6.2\n\n### Configuration\n\n[sqlfluff]\r\nverbose = 1\r\ndialect = snowflake\r\ntemplater = jinja\r\nexclude_rules = L027,L031,L032,L036,L044,L046,L034\r\noutput_line_length = 121\r\nsql_file_exts=.sql\r\n\r\n[sqlfluff:rules]\r\ntab_space_size = 4\r\nmax_line_length = 250\r\nindent_unit = space\r\ncomma_style = trailing\r\nallow_scalar = True\r\nsingle_table_references = consistent\r\nunquoted_identifiers_policy = aliases\r\n\r\n\r\n[sqlfluff:rules:L010] # Keywords\r\ncapitalisation_policy = upper\r\n\r\n[sqlfluff:rules:L014]\r\nextended_capitalisation_policy = lower\r\n\r\n[sqlfluff:rules:L030] # function names\r\ncapitalisation_policy = upper\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [ ] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "> Version\r\n> sqlfluff, version 0.6.2\r\n\r\nIs this correct? If so that is a VERY old version so please upgrade. Though confirmed this is still an issue in latest. But still, going to need to upgrade to get any fix for this.\n> > Version\r\n> > sqlfluff, version 0.6.2\r\n> \r\n> Is this correct? If so that is a VERY old version so please upgrade. Though confirmed this is still an issue in latest. But still, going to need to upgrade to get any fix for this.\r\n\r\nThanks for your response! I had sqlfluff globally installed with version 0.6.2 but i changed it now to 0.11.0 and still it is the same issue.\nThe rule probably needs updating to be \"template aware\". A few other rules have required similar updates and may provide useful inspiration for a fix.\r\n\r\n```\r\nsrc/sqlfluff/rules/L019.py\r\n140: and not last_seg.is_templated\r\n209: if last_seg.is_type(\"comma\") and not context.segment.is_templated:\r\n\r\nsrc/sqlfluff/rules/L003.py\r\n77: if elem.is_type(\"whitespace\") and elem.is_templated:\r\n148: templated_line = elem.is_templated\r\n\r\nsrc/sqlfluff/rules/L010.py\r\n87: if context.segment.is_templated:\r\n```\nI can't reproduce this issue with SQLFluff 0.11.0. This is the terminal output I get:\r\n```\r\n(sqlfluff-0.11.0) \u279c /tmp sqlfluff fix test.template.sql\r\n==== sqlfluff ====\r\nsqlfluff: 0.11.0 python: 3.9.1\r\nimplementation: cpython dialect: snowflake\r\nverbosity: 1 templater: jinja\r\n\r\n==== finding fixable violations ====\r\n=== [ path: test.template.sql ] ===\r\n\r\n== [test.template.sql] FAIL \r\nL: 2 | P: 1 | L003 | Indent expected and not found compared to line #1 \r\n==== fixing violations ====\r\n1 fixable linting violations found\r\nAre you sure you wish to attempt to fix these? [Y/n] ...\r\nAttempting fixes...\r\nPersisting Changes...\r\n== [test.template.sql] PASS\r\nDone. Please check your files to confirm.\r\nAll Finished \ud83d\udcdc \ud83c\udf89!\r\n```\r\n\r\nAnd this is the resulting file. SQLFluff indented line 2 but no newline was added.\r\n```\r\n{% if true %}\r\n SELECT 1 + 1\r\n{%- endif %}\r\n```\nI can @barrywhart but it only works when the final newline in the file doesn't exist.\r\n\r\nIf on mac you can run something like this to strip the final newline character:\r\n\r\n```\r\ntruncate -s -1 test.sql > test2.sql\r\n```\r\n\r\nThen fix `test2.sql` with default config and you'll see it.\nThere's a bug in `JinjaTracer` -- if a Jinja block (e.g. `{% endif %}` is the final slice in the file (i. there's no final newline), that slice is missing from the output. This will have to be fixed before we can fix L009, because at present, L009 cannot \"see\" that `{% endif %}` after the `1`.", "created_at": "2022-03-14T19:46:08Z", "version": "0.10", "FAIL_TO_PASS": ["test/api/simple_test.py::test__api__lint_string", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%-"], "PASS_TO_PASS": ["test/api/simple_test.py::test__api__lint_string_without_violations", "test/api/simple_test.py::test__api__lint_string_specific", "test/api/simple_test.py::test__api__lint_string_specific_single", "test/api/simple_test.py::test__api__lint_string_specific_exclude", "test/api/simple_test.py::test__api__lint_string_specific_exclude_single", "test/api/simple_test.py::test__api__lint_string_specific_exclude_all_failed_rules", "test/api/simple_test.py::test__api__fix_string", "test/api/simple_test.py::test__api__fix_string_specific", "test/api/simple_test.py::test__api__fix_string_specific_exclude", "test/api/simple_test.py::test__api__fix_string_unparsable", "test/api/simple_test.py::test__api__fix_string_unparsable_fix_even_unparsable", "test/api/simple_test.py::test__api__parse_string", "test/api/simple_test.py::test__api__parse_fail", "test/api/simple_test.py::test__api__config_path", "test/api/simple_test.py::test__api__config_override[kwargs0-expected0]", "test/api/simple_test.py::test__api__config_override[kwargs1-expected1]", "test/api/simple_test.py::test__api__invalid_dialect", "test/core/templaters/jinja_test.py::test__templater_jinja[simple]", "test/core/templaters/jinja_test.py::test__templater_jinja[unboundlocal_bugfix]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_left_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_right_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_comment]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[union_all_loop1]", "test/core/templaters/jinja_test.py::test_templater_set_block_handling", "test/core/templaters/jinja_test.py::test__templater_jinja_error_variable", "test/core/templaters/jinja_test.py::test__templater_jinja_error_syntax", "test/core/templaters/jinja_test.py::test__templater_jinja_error_catastrophic", "test/core/templaters/jinja_test.py::test__templater_jinja_error_macro_path_does_not_exist", "test/core/templaters/jinja_test.py::test__templater_jinja_lint_empty", "test/core/templaters/jinja_test.py::test__templater_full[jinja_a/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_b/jinja-False-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/var_default-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_e/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_f/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_g_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_h_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag_2-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_j_libraries/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_k_config_override_path_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/001-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/002-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_m_libraries_module/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_n_nested_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[{%", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[-None-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[foo-None-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{{", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT\\n", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[select\\n"], "environment_setup_commit": "3d52e8270d82aeccf4c516d059a80a6947919aea"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-2336", "base_commit": "37a993f7ad841ab3035d1db5ce6525f2e5584fd5", "patch": "diff --git a/src/sqlfluff/core/rules/analysis/select.py b/src/sqlfluff/core/rules/analysis/select.py\n--- a/src/sqlfluff/core/rules/analysis/select.py\n+++ b/src/sqlfluff/core/rules/analysis/select.py\n@@ -124,7 +124,7 @@ def _has_value_table_function(table_expr, dialect):\n # We need the dialect to get the value table function names. If\n # we don't have it, assume the clause does not have a value table\n # function.\n- return False\n+ return False # pragma: no cover\n \n for function_name in table_expr.recursive_crawl(\"function_name\"):\n # Other rules can increase whitespace in the function name, so use strip to remove\n@@ -138,7 +138,7 @@ def _get_pivot_table_columns(segment, dialect):\n if not dialect:\n # We need the dialect to get the pivot table column names. If\n # we don't have it, assume the clause does not have a pivot table\n- return []\n+ return [] # pragma: no cover\n \n fc = segment.get_child(\"from_pivot_expression\")\n if not fc:\ndiff --git a/src/sqlfluff/core/rules/reference.py b/src/sqlfluff/core/rules/reference.py\nnew file mode 100644\n--- /dev/null\n+++ b/src/sqlfluff/core/rules/reference.py\n@@ -0,0 +1,26 @@\n+\"\"\"Components for working with object and table references.\"\"\"\n+from typing import Sequence, Tuple\n+\n+\n+def object_ref_matches_table(\n+ possible_references: Sequence[Tuple[str, ...]], targets: Sequence[Tuple[str, ...]]\n+) -> bool:\n+ \"\"\"Return True if any of the possible references matches a target.\"\"\"\n+ # Simple case: If there are no references, assume okay\n+ # (i.e. no mismatch = good).\n+ if not possible_references:\n+ return True\n+ # Simple case: Reference exactly matches a target.\n+ if any(pr in targets for pr in possible_references):\n+ return True\n+ # Tricky case: If one is shorter than the other, check for a suffix match.\n+ # (Note this is an \"optimistic\" check, i.e. it assumes the ignored parts of\n+ # the target don't matter. In a SQL context, this is basically assuming\n+ # there was an earlier \"USE <>\" or similar directive.\n+ for pr in possible_references:\n+ for t in targets:\n+ if (len(pr) < len(t) and pr == t[-len(pr) :]) or (\n+ len(t) < len(pr) and t == pr[-len(t) :]\n+ ):\n+ return True\n+ return False\ndiff --git a/src/sqlfluff/dialects/dialect_ansi.py b/src/sqlfluff/dialects/dialect_ansi.py\n--- a/src/sqlfluff/dialects/dialect_ansi.py\n+++ b/src/sqlfluff/dialects/dialect_ansi.py\n@@ -734,6 +734,18 @@ def extract_possible_references(\n return [refs[-level]]\n return []\n \n+ def extract_possible_multipart_references(\n+ self, levels: List[Union[ObjectReferenceLevel, int]]\n+ ) -> List[Tuple[ObjectReferencePart, ...]]:\n+ \"\"\"Extract possible multipart references, e.g. schema.table.\"\"\"\n+ levels_tmp = [self._level_to_int(level) for level in levels]\n+ min_level = min(levels_tmp)\n+ max_level = max(levels_tmp)\n+ refs = list(self.iter_raw_references())\n+ if len(refs) >= max_level:\n+ return [tuple(refs[-max_level : 1 - min_level])]\n+ return []\n+\n @staticmethod\n def _level_to_int(level: Union[ObjectReferenceLevel, int]) -> int:\n # If it's an ObjectReferenceLevel, get the value. Otherwise, assume it's\n@@ -1156,7 +1168,6 @@ def get_eventual_alias(self) -> Optional[AliasInfo]:\n return AliasInfo(segment.raw, segment, True, self, alias_expression, ref)\n \n # If not return the object name (or None if there isn't one)\n- # ref = self.get_child(\"object_reference\")\n if ref:\n # Return the last element of the reference.\n penultimate_ref: ObjectReferenceSegment.ObjectReferencePart = list(\ndiff --git a/src/sqlfluff/dialects/dialect_bigquery.py b/src/sqlfluff/dialects/dialect_bigquery.py\n--- a/src/sqlfluff/dialects/dialect_bigquery.py\n+++ b/src/sqlfluff/dialects/dialect_bigquery.py\n@@ -740,6 +740,18 @@ def extract_possible_references(self, level):\n return [refs[1], refs[2]]\n return super().extract_possible_references(level) # pragma: no cover\n \n+ def extract_possible_multipart_references(self, levels):\n+ \"\"\"Extract possible multipart references, e.g. schema.table.\"\"\"\n+ levels_tmp = [self._level_to_int(level) for level in levels]\n+ min_level = min(levels_tmp)\n+ max_level = max(levels_tmp)\n+ refs = list(self.iter_raw_references())\n+ if max_level == self.ObjectReferenceLevel.SCHEMA.value and len(refs) >= 3:\n+ return [tuple(refs[0 : max_level - min_level + 1])]\n+ # Note we aren't handling other possible cases. We'll add these as\n+ # needed.\n+ return super().extract_possible_multipart_references(levels)\n+\n \n @bigquery_dialect.segment()\n class HyphenatedObjectReferenceSegment(ObjectReferenceSegment): # type: ignore\ndiff --git a/src/sqlfluff/rules/L025.py b/src/sqlfluff/rules/L025.py\n--- a/src/sqlfluff/rules/L025.py\n+++ b/src/sqlfluff/rules/L025.py\n@@ -85,16 +85,18 @@ def _eval(self, context: RuleContext) -> EvalResultType:\n def _analyze_table_aliases(cls, query: L025Query, dialect: Dialect):\n # Get table aliases defined in query.\n for selectable in query.selectables:\n- select_info = get_select_statement_info(selectable.selectable, dialect)\n+ select_info = selectable.select_info\n if select_info:\n # Record the aliases.\n query.aliases += select_info.table_aliases\n \n- # Look at each table reference; if its an alias reference,\n+ # Look at each table reference; if it's an alias reference,\n # resolve the alias: could be an alias defined in \"query\"\n # itself or an \"ancestor\" query.\n for r in select_info.reference_buffer:\n- for tr in r.extract_possible_references(level=r.ObjectReferenceLevel.TABLE): # type: ignore\n+ for tr in r.extract_possible_references(\n+ level=r.ObjectReferenceLevel.TABLE\n+ ):\n # This function walks up the query's parent stack if necessary.\n cls._resolve_and_mark_reference(query, tr.part)\n \ndiff --git a/src/sqlfluff/rules/L026.py b/src/sqlfluff/rules/L026.py\n--- a/src/sqlfluff/rules/L026.py\n+++ b/src/sqlfluff/rules/L026.py\n@@ -1,13 +1,33 @@\n \"\"\"Implementation of Rule L026.\"\"\"\n-\n-from sqlfluff.core.rules.analysis.select import get_aliases_from_select\n-from sqlfluff.core.rules.base import EvalResultType, LintResult, RuleContext\n+from dataclasses import dataclass, field\n+from typing import cast, List, Optional, Tuple\n+\n+from sqlfluff.core.dialects.base import Dialect\n+from sqlfluff.core.rules.analysis.select_crawler import (\n+ Query as SelectCrawlerQuery,\n+ SelectCrawler,\n+)\n+from sqlfluff.core.dialects.common import AliasInfo\n+from sqlfluff.core.rules.base import (\n+ BaseRule,\n+ LintResult,\n+ RuleContext,\n+ EvalResultType,\n+)\n+from sqlfluff.core.rules.functional import sp\n from sqlfluff.core.rules.doc_decorators import document_configuration\n-from sqlfluff.rules.L020 import Rule_L020\n+from sqlfluff.core.rules.reference import object_ref_matches_table\n+\n+\n+@dataclass\n+class L026Query(SelectCrawlerQuery):\n+ \"\"\"SelectCrawler Query with custom L026 info.\"\"\"\n+\n+ aliases: List[AliasInfo] = field(default_factory=list)\n \n \n @document_configuration\n-class Rule_L026(Rule_L020):\n+class Rule_L026(BaseRule):\n \"\"\"References cannot reference objects not present in ``FROM`` clause.\n \n NB: This rule is disabled by default for BigQuery due to its use of\n@@ -36,61 +56,7 @@ class Rule_L026(Rule_L020):\n \n config_keywords = [\"force_enable\"]\n \n- @staticmethod\n- def _is_bad_tbl_ref(table_aliases, parent_select, tbl_ref):\n- \"\"\"Given a table reference, try to find what it's referring to.\"\"\"\n- # Is it referring to one of the table aliases?\n- if tbl_ref[0] in [a.ref_str for a in table_aliases]:\n- # Yes. Therefore okay.\n- return False\n-\n- # Not a table alias. It it referring to a correlated subquery?\n- if parent_select:\n- parent_aliases, _ = get_aliases_from_select(parent_select)\n- if parent_aliases and tbl_ref[0] in [a[0] for a in parent_aliases]:\n- # Yes. Therefore okay.\n- return False\n-\n- # It's not referring to an alias or a correlated subquery. Looks like a\n- # bad reference (i.e. referring to something unknown.)\n- return True\n-\n- def _lint_references_and_aliases(\n- self,\n- table_aliases,\n- standalone_aliases,\n- references,\n- col_aliases,\n- using_cols,\n- parent_select,\n- ):\n- # A buffer to keep any violations.\n- violation_buff = []\n-\n- # Check all the references that we have, do they reference present aliases?\n- for r in references:\n- tbl_refs = r.extract_possible_references(level=r.ObjectReferenceLevel.TABLE)\n- if tbl_refs and all(\n- self._is_bad_tbl_ref(table_aliases, parent_select, tbl_ref)\n- for tbl_ref in tbl_refs\n- ):\n- violation_buff.append(\n- LintResult(\n- # Return the first segment rather than the string\n- anchor=tbl_refs[0].segments[0],\n- description=f\"Reference {r.raw!r} refers to table/view \"\n- \"not found in the FROM clause or found in parent \"\n- \"subquery.\",\n- )\n- )\n- return violation_buff or None\n-\n def _eval(self, context: RuleContext) -> EvalResultType:\n- \"\"\"Override Rule L020 for dialects that use structs.\n-\n- Some dialects use structs (e.g. column.field) which look like\n- table references and so incorrectly trigger this rule.\n- \"\"\"\n # Config type hints\n self.force_enable: bool\n \n@@ -100,4 +66,128 @@ def _eval(self, context: RuleContext) -> EvalResultType:\n ):\n return LintResult()\n \n- return super()._eval(context=context)\n+ violations: List[LintResult] = []\n+ start_types = [\"select_statement\", \"delete_statement\", \"update_statement\"]\n+ if context.segment.is_type(\n+ *start_types\n+ ) and not context.functional.parent_stack.any(sp.is_type(*start_types)):\n+ dml_target_table: Optional[Tuple[str, ...]] = None\n+ if not context.segment.is_type(\"select_statement\"):\n+ # Extract first table reference. This will be the target\n+ # table in a DELETE or UPDATE statement.\n+ table_reference = next(\n+ context.segment.recursive_crawl(\"table_reference\"), None\n+ )\n+ if table_reference:\n+ dml_target_table = self._table_ref_as_tuple(table_reference)\n+\n+ # Verify table references in any SELECT statements found in or\n+ # below context.segment in the parser tree.\n+ crawler = SelectCrawler(\n+ context.segment, context.dialect, query_class=L026Query\n+ )\n+ query: L026Query = cast(L026Query, crawler.query_tree)\n+ self._analyze_table_references(\n+ query, dml_target_table, context.dialect, violations\n+ )\n+ return violations or None\n+\n+ @classmethod\n+ def _alias_info_as_tuples(cls, alias_info: AliasInfo) -> List[Tuple[str, ...]]:\n+ result: List[Tuple[str, ...]] = []\n+ if alias_info.aliased:\n+ result.append((alias_info.ref_str,))\n+ if alias_info.object_reference:\n+ result.append(cls._table_ref_as_tuple(alias_info.object_reference))\n+ return result\n+\n+ @staticmethod\n+ def _table_ref_as_tuple(table_reference) -> Tuple[str, ...]:\n+ return tuple(ref.part for ref in table_reference.iter_raw_references())\n+\n+ def _analyze_table_references(\n+ self,\n+ query: L026Query,\n+ dml_target_table: Optional[Tuple[str, ...]],\n+ dialect: Dialect,\n+ violations: List[LintResult],\n+ ):\n+ # For each query...\n+ for selectable in query.selectables:\n+ select_info = selectable.select_info\n+ if select_info:\n+ # Record the available tables.\n+ query.aliases += select_info.table_aliases\n+\n+ # Try and resolve each reference to a value in query.aliases (or\n+ # in an ancestor query).\n+ for r in select_info.reference_buffer:\n+ # This function walks up the query's parent stack if necessary.\n+ violation = self._resolve_reference(\n+ r, self._get_table_refs(r, dialect), dml_target_table, query\n+ )\n+ if violation:\n+ violations.append(violation)\n+\n+ # Visit children.\n+ for child in query.children:\n+ self._analyze_table_references(\n+ cast(L026Query, child), dml_target_table, dialect, violations\n+ )\n+\n+ @staticmethod\n+ def _get_table_refs(ref, dialect):\n+ \"\"\"Given ObjectReferenceSegment, determine possible table references.\"\"\"\n+ tbl_refs = []\n+ # First, handle any schema.table references.\n+ for sr, tr in ref.extract_possible_multipart_references(\n+ levels=[\n+ ref.ObjectReferenceLevel.SCHEMA,\n+ ref.ObjectReferenceLevel.TABLE,\n+ ]\n+ ):\n+ tbl_refs.append((tr, (sr.part, tr.part)))\n+ # Maybe check for simple table references. Two cases:\n+ # - For most dialects, skip this if it's a schema+table reference -- the\n+ # reference was specific, so we shouldn't ignore that by looking\n+ # elsewhere.)\n+ # - Always do this in BigQuery. BigQuery table references are frequently\n+ # ambiguous because BigQuery SQL supports structures, making some\n+ # multi-level \".\" references impossible to interpret with certainty.\n+ # We may need to genericize this code someday to support other\n+ # dialects. If so, this check should probably align somehow with\n+ # whether the dialect overrides\n+ # ObjectReferenceSegment.extract_possible_references().\n+ if not tbl_refs or dialect.name in [\"bigquery\"]:\n+ for tr in ref.extract_possible_references(\n+ level=ref.ObjectReferenceLevel.TABLE\n+ ):\n+ tbl_refs.append((tr, (tr.part,)))\n+ return tbl_refs\n+\n+ def _resolve_reference(\n+ self, r, tbl_refs, dml_target_table: Optional[Tuple[str, ...]], query: L026Query\n+ ):\n+ # Does this query define the referenced table?\n+ possible_references = [tbl_ref[1] for tbl_ref in tbl_refs]\n+ targets = []\n+ for alias in query.aliases:\n+ targets += self._alias_info_as_tuples(alias)\n+ if not object_ref_matches_table(possible_references, targets):\n+ # No. Check the parent query, if there is one.\n+ if query.parent:\n+ return self._resolve_reference(\n+ r, tbl_refs, dml_target_table, cast(L026Query, query.parent)\n+ )\n+ # No parent query. If there's a DML statement at the root, check its\n+ # target table.\n+ elif not dml_target_table or not object_ref_matches_table(\n+ possible_references, [dml_target_table]\n+ ):\n+ return LintResult(\n+ # Return the first segment rather than the string\n+ anchor=tbl_refs[0][0].segments[0],\n+ description=f\"Reference {r.raw!r} refers to table/view \"\n+ \"not found in the FROM clause or found in ancestor \"\n+ \"statement.\",\n+ )\n", "test_patch": "diff --git a/test/core/rules/reference_test.py b/test/core/rules/reference_test.py\nnew file mode 100644\n--- /dev/null\n+++ b/test/core/rules/reference_test.py\n@@ -0,0 +1,72 @@\n+\"\"\"Test components for working with object and table references.\"\"\"\n+import pytest\n+\n+from sqlfluff.core.rules import reference\n+\n+\n+@pytest.mark.parametrize(\n+ \"possible_references, targets, result\",\n+ [\n+ # Empty list of references is always True.\n+ [[], [(\"abc\",)], True],\n+ # Simple cases: one reference, one target.\n+ [[(\"agent1\",)], [(\"agent1\",)], True],\n+ [[(\"agent1\",)], [(\"customer\",)], False],\n+ # Multiple references. If any match, good.\n+ [[(\"bar\",), (\"user_id\",)], [(\"bar\",)], True],\n+ [[(\"foo\",), (\"user_id\",)], [(\"bar\",)], False],\n+ # Multiple targets. If any reference matches, good.\n+ [[(\"table1\",)], [(\"table1\",), (\"table2\",), (\"table3\",)], True],\n+ [[(\"tbl2\",)], [(\"db\", \"sc\", \"tbl1\")], False],\n+ [[(\"tbl2\",)], [(\"db\", \"sc\", \"tbl2\")], True],\n+ # Multi-part references and targets. If one tuple is shorter than\n+ # the other, checks for a suffix match.\n+ [\n+ [\n+ (\n+ \"rc\",\n+ \"tbl1\",\n+ )\n+ ],\n+ [(\"db\", \"sc\", \"tbl1\")],\n+ False,\n+ ],\n+ [\n+ [\n+ (\n+ \"sc\",\n+ \"tbl1\",\n+ )\n+ ],\n+ [(\"db\", \"sc\", \"tbl1\")],\n+ True,\n+ ],\n+ [\n+ [\n+ (\n+ \"cb\",\n+ \"sc\",\n+ \"tbl1\",\n+ )\n+ ],\n+ [(\"db\", \"sc\", \"tbl1\")],\n+ False,\n+ ],\n+ [\n+ [\n+ (\n+ \"db\",\n+ \"sc\",\n+ \"tbl1\",\n+ )\n+ ],\n+ [(\"db\", \"sc\", \"tbl1\")],\n+ True,\n+ ],\n+ [[(\"public\", \"agent1\")], [(\"agent1\",)], True],\n+ [[(\"public\", \"agent1\")], [(\"public\",)], False],\n+ ],\n+)\n+def test_object_ref_matches_table(possible_references, targets, result):\n+ \"\"\"Test object_ref_matches_table().\"\"\"\n+ assert reference.object_ref_matches_table(possible_references, targets) == result\ndiff --git a/test/fixtures/rules/std_rule_cases/L026.yml b/test/fixtures/rules/std_rule_cases/L026.yml\n--- a/test/fixtures/rules/std_rule_cases/L026.yml\n+++ b/test/fixtures/rules/std_rule_cases/L026.yml\n@@ -110,3 +110,64 @@ test_pass_object_referenced_6:\n table3\n on table2.y_id = table3.y_id\n ) as cc\n+\n+test_pass_object_referenced_7:\n+ pass_str: |\n+ UPDATE my_table\n+ SET row_sum = (\n+ SELECT COUNT(*) AS row_sum\n+ FROM\n+ another_table\n+ WHERE\n+ another_table.id = my_table.id\n+ )\n+\n+test_fail_object_referenced_7:\n+ fail_str: |\n+ UPDATE my_table\n+ SET row_sum = (\n+ SELECT COUNT(*) AS row_sum\n+ FROM\n+ another_table\n+ WHERE\n+ another_table.id = my_tableeee.id\n+ )\n+\n+test_pass_object_referenced_8:\n+ pass_str: |\n+ DELETE FROM agent1\n+ WHERE EXISTS(\n+ SELECT customer.cust_id FROM customer\n+ WHERE agent1.agent_code <> customer.agent_code);\n+\n+test_pass_two_part_reference_8:\n+ pass_str: |\n+ delete from public.agent1\n+ where exists(\n+ select customer.cust_id from customer\n+ where agent1.agent_code <> customer.agent_code)\n+\n+test_pass_two_part_reference_9:\n+ pass_str: |\n+ delete from public.agent1\n+ where exists(\n+ select customer.cust_id from customer\n+ where public.agent1.agent_code <> customer.agent_code)\n+\n+test_fail_two_part_reference_10:\n+ fail_str: |\n+ select *\n+ from schema1.agent1\n+ where schema2.agent1.agent_code <> 'abc'\n+\n+test_fail_two_part_reference_11:\n+ fail_str: |\n+ delete from schema1.agent1\n+ where exists(\n+ select customer.cust_id from customer\n+ where schema2.agent1.agent_code <> customer.agent_code)\n+\n+test_pass_two_part_reference_11:\n+ pass_str: |\n+ select * from agent1\n+ where public.agent1.agent_code <> '3'\n", "problem_statement": "L026: Rule incorrectly flag column does not exist in `FROM` clause in an UPDATE statement.\n## Expected Behaviour\r\n\r\nL026 should not fail when a subquery in an UPDATE statement references a column from the UPDATE target.\r\n\r\n## Observed Behaviour\r\n\r\nL026 failed due to reference was not found in the FROM clause with the following error printed (When using `sample.sql` content below)\r\n\r\n```\r\nL: 7 | P: 28 | L026 | Reference 'my_table.id' refers to table/view not found\r\n | in the FROM clause or found in parent subquery.\r\n```\r\n\r\n## Steps to Reproduce\r\n\r\n1. Create `sample.sql` with the content below\r\n```\r\nUPDATE my_table\r\nSET row_sum = (\r\n SELECT COUNT(*) AS row_sum\r\n FROM\r\n another_table\r\n WHERE\r\n another_table.id = my_table.id\r\n);\r\n```\r\n2. Run SQLFluff by `sqlfluff lint sample.sql`\r\n\r\n## Dialect\r\n\r\nDefault / Ansi (No dialect specified)\r\n\r\n## Version\r\n```\r\n(.venv) ~/code/sqlfluff (main) $ sqlfluff --version\r\nsqlfluff, version 0.9.0\r\n```\r\n\r\n```\r\n(.venv) ~/code/sqlfluff (main) $ python --version\r\nPython 3.9.9\r\n```\r\n\r\n## Configuration\r\nDefault. No customization.\r\n\n", "hints_text": "", "created_at": "2022-01-17T21:35:10Z", "version": "0.8", "FAIL_TO_PASS": ["test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references0-targets0-True]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references1-targets1-True]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references2-targets2-False]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references3-targets3-True]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references4-targets4-False]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references5-targets5-True]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references6-targets6-False]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references7-targets7-True]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references8-targets8-False]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references9-targets9-True]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references10-targets10-False]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references11-targets11-True]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references12-targets12-True]", "test/core/rules/reference_test.py::test_object_ref_matches_table[possible_references13-targets13-False]"], "PASS_TO_PASS": [], "environment_setup_commit": "a5c4eae4e3e419fe95460c9afd9cf39a35a470c4"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-5074", "base_commit": "7b7fd603a19755a9f3707ebbf95d18ee635716d8", "patch": "diff --git a/src/sqlfluff/core/errors.py b/src/sqlfluff/core/errors.py\n--- a/src/sqlfluff/core/errors.py\n+++ b/src/sqlfluff/core/errors.py\n@@ -47,9 +47,15 @@ def __init__(\n self.line_pos = line_pos\n super().__init__(self.desc())\n \n+ def __eq__(self, other) -> bool:\n+ \"\"\"Errors compare equal if they are the same type and same content.\"\"\"\n+ if not isinstance(other, self.__class__):\n+ return False\n+ return self.__dict__ == other.__dict__\n+\n def __reduce__(\n self,\n- ) -> Tuple[Type[\"SQLBaseError\"], Tuple[Any, ...]]: # pragma: no cover\n+ ) -> Tuple[Type[\"SQLBaseError\"], Tuple[Any, ...]]:\n \"\"\"Prepare the SQLBaseError for pickling.\"\"\"\n return type(self), (\n self.description,\n@@ -169,6 +175,9 @@ def __init__(\n segment: Optional[\"BaseSegment\"] = None,\n line_no: int = 0,\n line_pos: int = 0,\n+ ignore: bool = False,\n+ fatal: bool = False,\n+ warning: Optional[bool] = None,\n ) -> None:\n # Store the segment on creation - we might need it later\n self.segment = segment\n@@ -177,13 +186,24 @@ def __init__(\n pos=segment.pos_marker if segment else None,\n line_no=line_no,\n line_pos=line_pos,\n+ ignore=ignore,\n+ fatal=fatal,\n+ warning=warning,\n )\n \n def __reduce__(\n self,\n- ) -> Tuple[Type[\"SQLParseError\"], Tuple[Any, ...]]: # pragma: no cover\n+ ) -> Tuple[Type[\"SQLParseError\"], Tuple[Any, ...]]:\n \"\"\"Prepare the SQLParseError for pickling.\"\"\"\n- return type(self), (self.description, self.segment, self.line_no, self.line_pos)\n+ return type(self), (\n+ self.description,\n+ self.segment,\n+ self.line_no,\n+ self.line_pos,\n+ self.ignore,\n+ self.fatal,\n+ self.warning,\n+ )\n \n \n class SQLLintError(SQLBaseError):\n@@ -208,20 +228,34 @@ def __init__(\n segment: \"BaseSegment\",\n rule: \"BaseRule\",\n fixes: Optional[List[\"LintFix\"]] = None,\n+ ignore: bool = False,\n+ fatal: bool = False,\n+ warning: Optional[bool] = None,\n ) -> None:\n- # Something about position, message and fix?\n self.segment = segment\n self.rule = rule\n self.fixes = fixes or []\n super().__init__(\n- description=description, pos=segment.pos_marker if segment else None\n+ description=description,\n+ pos=segment.pos_marker if segment else None,\n+ ignore=ignore,\n+ fatal=fatal,\n+ warning=warning,\n )\n \n def __reduce__(\n self,\n- ) -> Tuple[Type[\"SQLLintError\"], Tuple[Any, ...]]: # pragma: no cover\n+ ) -> Tuple[Type[\"SQLLintError\"], Tuple[Any, ...]]:\n \"\"\"Prepare the SQLLintError for pickling.\"\"\"\n- return type(self), (self.description, self.segment, self.rule, self.fixes)\n+ return type(self), (\n+ self.description,\n+ self.segment,\n+ self.rule,\n+ self.fixes,\n+ self.ignore,\n+ self.fatal,\n+ self.warning,\n+ )\n \n @property\n def fixable(self) -> bool:\ndiff --git a/src/sqlfluff/core/parser/markers.py b/src/sqlfluff/core/parser/markers.py\n--- a/src/sqlfluff/core/parser/markers.py\n+++ b/src/sqlfluff/core/parser/markers.py\n@@ -62,6 +62,11 @@ def __ge__(self, other: \"PositionMarker\") -> bool:\n def __le__(self, other: \"PositionMarker\") -> bool:\n return self.working_loc <= other.working_loc # pragma: no cover TODO?\n \n+ def __eq__(self, other) -> bool:\n+ if not isinstance(other, PositionMarker):\n+ return False # pragma: no cover\n+ return self.working_loc == other.working_loc\n+\n @property\n def working_loc(self) -> Tuple[int, int]:\n \"\"\"Location tuple for the working position.\"\"\"\n", "test_patch": "diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py\n--- a/test/cli/commands_test.py\n+++ b/test/cli/commands_test.py\n@@ -379,6 +379,20 @@ def test__cli__command_render_stdin():\n \"test/fixtures/linter/operator_errors.sql\",\n ],\n ),\n+ # Check ignoring linting (multiprocess)\n+ # https://github.com/sqlfluff/sqlfluff/issues/5066\n+ (\n+ lint,\n+ [\n+ \"-n\",\n+ \"--ignore\",\n+ \"linting\",\n+ \"-p\",\n+ \"2\",\n+ \"test/fixtures/linter/operator_errors.sql\",\n+ \"test/fixtures/linter/comma_errors.sql\",\n+ ],\n+ ),\n # Check linting works in specifying multiple rules\n (\n lint,\ndiff --git a/test/core/errors_test.py b/test/core/errors_test.py\nnew file mode 100644\n--- /dev/null\n+++ b/test/core/errors_test.py\n@@ -0,0 +1,75 @@\n+\"\"\"Tests pickling and unpickling of errors.\"\"\"\n+\n+import pickle\n+import pytest\n+import copy\n+\n+from sqlfluff.core.parser import PositionMarker, RawSegment\n+from sqlfluff.core.rules import BaseRule\n+from sqlfluff.core.templaters import TemplatedFile\n+\n+from sqlfluff.core.errors import SQLBaseError, SQLLintError, SQLParseError, SQLLexError\n+\n+\n+class Rule_T078(BaseRule):\n+ \"\"\"A dummy rule.\"\"\"\n+\n+ groups = (\"all\",)\n+\n+ def _eval(self, context):\n+ pass\n+\n+\n+def assert_pickle_robust(err: SQLBaseError):\n+ \"\"\"Test that the class remains the same through copying and pickling.\"\"\"\n+ # First try copying (and make sure they still compare equal)\n+ err_copy = copy.copy(err)\n+ assert err_copy == err\n+ # Then try picking (and make sure they also still compare equal)\n+ pickled = pickle.dumps(err)\n+ pickle_copy = pickle.loads(pickled)\n+ assert pickle_copy == err\n+\n+\n+@pytest.mark.parametrize(\n+ \"ignore\",\n+ [True, False],\n+)\n+def test__lex_error_pickle(ignore):\n+ \"\"\"Test lexing error pickling.\"\"\"\n+ template = TemplatedFile.from_string(\"foobar\")\n+ err = SQLLexError(\"Foo\", pos=PositionMarker(slice(0, 6), slice(0, 6), template))\n+ # Set ignore to true if configured.\n+ # NOTE: This not copying was one of the reasons for this test.\n+ err.ignore = ignore\n+ assert_pickle_robust(err)\n+\n+\n+@pytest.mark.parametrize(\n+ \"ignore\",\n+ [True, False],\n+)\n+def test__parse_error_pickle(ignore):\n+ \"\"\"Test parse error pickling.\"\"\"\n+ template = TemplatedFile.from_string(\"foobar\")\n+ segment = RawSegment(\"foobar\", PositionMarker(slice(0, 6), slice(0, 6), template))\n+ err = SQLParseError(\"Foo\", segment=segment)\n+ # Set ignore to true if configured.\n+ # NOTE: This not copying was one of the reasons for this test.\n+ err.ignore = ignore\n+ assert_pickle_robust(err)\n+\n+\n+@pytest.mark.parametrize(\n+ \"ignore\",\n+ [True, False],\n+)\n+def test__lint_error_pickle(ignore):\n+ \"\"\"Test lint error pickling.\"\"\"\n+ template = TemplatedFile.from_string(\"foobar\")\n+ segment = RawSegment(\"foobar\", PositionMarker(slice(0, 6), slice(0, 6), template))\n+ err = SQLLintError(\"Foo\", segment=segment, rule=Rule_T078)\n+ # Set ignore to true if configured.\n+ # NOTE: This not copying was one of the reasons for this test.\n+ err.ignore = ignore\n+ assert_pickle_robust(err)\n", "problem_statement": "Inconsistent output depending on --processes flag when --ignore linting is used\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nDepending on the value you set for the `--processes` flag when also using `--ignore linting`, different output with different exit codes are generated.\n\n### Expected Behaviour\n\nThe same exit code should be generated, independently of the `--processes` flag. Furthermore, from https://docs.sqlfluff.com/en/stable/production.html#using-sqlfluff-on-a-whole-sql-codebase I would expect that exit codes should be either `0` or `65`, not `1`.\n\n### Observed Behaviour\n\nSee the How to reproduce section.\n\n### How to reproduce\n\nCreate a `test.sql` file with the following content:\r\n\r\n```SQL\r\nCREATE TABLE example (\r\n id TEXT DEFAULT 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. In condimentum congue est, ac orci aliquam.' PRIMARY KEY\r\n);\r\n```\r\n\r\nThe line is too long according to SQLFluff, caused by the large default value, so let's see the the output of SQLFluff.\r\n\r\nRunning\r\n\r\n```SHELL\r\nsqlfluff fix --dialect postgres --ignore linting --processes 2\r\n```\r\n\r\nresults in \r\n\r\n```\r\n==== finding fixable violations ====\r\n==== no fixable linting violations found ==== \r\nAll Finished \ud83d\udcdc \ud83c\udf89!\r\n [1 unfixable linting violations found]\r\n```\r\n\r\nwith exit code `1`. Running the same with one process instead:\r\n\r\n```SHELL\r\nsqlfluff fix --dialect postgres --ignore linting --processes 1\r\n```\r\n\r\nresults in\r\n\r\n```\r\n==== finding fixable violations ====\r\n==== no fixable linting violations found ==== \r\nAll Finished \ud83d\udcdc \ud83c\udf89!\r\n```\r\n\r\nand exit code `0`\r\n\r\nSame behaviour for `lint` and `format` commands.\n\n### Dialect\n\nPostgres\n\n### Version\n\n2.2.0, Python 3.10.6\n\n### Configuration\n\nNone, it's all in the CLI flags.\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [ ] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "This is _very_ interesting! I'll pick this one up \ud83d\udc4d \nIt's worth calling out that I think part of the problem here is that the docs are also out of date - but you're still right that the return codes should be _the same_ regardless of the `processes` setting.", "created_at": "2023-08-08T23:31:59Z", "version": "2.1", "FAIL_TO_PASS": ["test/cli/commands_test.py::test__cli__command_lint_parse[command19]", "test/core/errors_test.py::test__lex_error_pickle[True]", "test/core/errors_test.py::test__lex_error_pickle[False]", "test/core/errors_test.py::test__parse_error_pickle[True]", "test/core/errors_test.py::test__parse_error_pickle[False]", "test/core/errors_test.py::test__lint_error_pickle[True]", "test/core/errors_test.py::test__lint_error_pickle[False]"], "PASS_TO_PASS": ["test/cli/commands_test.py::test__cli__command_directed", "test/cli/commands_test.py::test__cli__command_dialect", "test/cli/commands_test.py::test__cli__command_no_dialect", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_explicit_warning", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_implicit_warning", "test/cli/commands_test.py::test__cli__command_dialect_legacy", "test/cli/commands_test.py::test__cli__command_extra_config_fail", "test/cli/commands_test.py::test__cli__command_lint_stdin[command0]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command1]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command2]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command3]", "test/cli/commands_test.py::test__cli__command_lint_empty_stdin", "test/cli/commands_test.py::test__cli__command_render_stdin", "test/cli/commands_test.py::test__cli__command_lint_parse[command0]", "test/cli/commands_test.py::test__cli__command_lint_parse[command1]", "test/cli/commands_test.py::test__cli__command_lint_parse[command2]", "test/cli/commands_test.py::test__cli__command_lint_parse[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command4]", "test/cli/commands_test.py::test__cli__command_lint_parse[command5]", "test/cli/commands_test.py::test__cli__command_lint_parse[command6]", "test/cli/commands_test.py::test__cli__command_lint_parse[command7]", "test/cli/commands_test.py::test__cli__command_lint_parse[command8]", "test/cli/commands_test.py::test__cli__command_lint_parse[command9]", "test/cli/commands_test.py::test__cli__command_lint_parse[command10]", "test/cli/commands_test.py::test__cli__command_lint_parse[command11]", "test/cli/commands_test.py::test__cli__command_lint_parse[command12]", "test/cli/commands_test.py::test__cli__command_lint_parse[command13]", "test/cli/commands_test.py::test__cli__command_lint_parse[command14]", "test/cli/commands_test.py::test__cli__command_lint_parse[command15]", "test/cli/commands_test.py::test__cli__command_lint_parse[command16]", "test/cli/commands_test.py::test__cli__command_lint_parse[command17]", "test/cli/commands_test.py::test__cli__command_lint_parse[command18]", "test/cli/commands_test.py::test__cli__command_lint_parse[command20]", "test/cli/commands_test.py::test__cli__command_lint_parse[command21]", "test/cli/commands_test.py::test__cli__command_lint_parse[command22]", "test/cli/commands_test.py::test__cli__command_lint_parse[command23]", "test/cli/commands_test.py::test__cli__command_lint_parse[command24]", "test/cli/commands_test.py::test__cli__command_lint_parse[command25]", "test/cli/commands_test.py::test__cli__command_lint_parse[command26]", "test/cli/commands_test.py::test__cli__command_lint_parse[command27]", "test/cli/commands_test.py::test__cli__command_lint_parse[command28]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command0-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command1-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command2-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command3-0]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command4-0]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command5-2]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command6-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command7-0]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command8-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command9-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command10-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command11-2]", "test/cli/commands_test.py::test__cli__command_lint_warning_explicit_file_ignored", "test/cli/commands_test.py::test__cli__command_lint_skip_ignore_files", "test/cli/commands_test.py::test__cli__command_lint_ignore_local_config", "test/cli/commands_test.py::test__cli__command_lint_warning", "test/cli/commands_test.py::test__cli__command_versioning", "test/cli/commands_test.py::test__cli__command_version", "test/cli/commands_test.py::test__cli__command_rules", "test/cli/commands_test.py::test__cli__command_dialects", "test/cli/commands_test.py::test__cli__command__fix[LT01-test/fixtures/linter/indentation_errors.sql0]", "test/cli/commands_test.py::test__cli__command__fix[LT01-test/fixtures/linter/whitespace_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[LT01-test/fixtures/linter/indentation_errors.sql1]", "test/cli/commands_test.py::test__cli__command__fix[LT02-test/fixtures/linter/indentation_error_hard.sql]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_templating_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error_FIX_EVEN_UNPARSABLE]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[2_files_with_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-True]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-True]", "test/cli/commands_test.py::test__cli__fix_loop_limit_behavior[--", "test/cli/commands_test.py::test__cli__command_fix_stdin[select", "test/cli/commands_test.py::test__cli__command_fix_stdin[", "test/cli/commands_test.py::test__cli__command_format_stdin[select", "test/cli/commands_test.py::test__cli__command_format_stdin[", "test/cli/commands_test.py::test__cli__command_fix_stdin_logging_to_stderr", "test/cli/commands_test.py::test__cli__command_fix_stdin_safety", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[create", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[select", "test/cli/commands_test.py::test__cli__command__fix_no_force[LT01-test/fixtures/linter/indentation_errors.sql-y-0-0]", "test/cli/commands_test.py::test__cli__command__fix_no_force[LT01-test/fixtures/linter/indentation_errors.sql-n-1-1]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-json]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[select", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[SElect", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command0]", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command1]", "test/cli/commands_test.py::test__cli__command_lint_nocolor", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-none]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-none]", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation_native", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation-native]", "test/cli/commands_test.py::test___main___help", "test/cli/commands_test.py::test_encoding[utf-8-ascii]", "test/cli/commands_test.py::test_encoding[utf-8-sig-UTF-8-SIG]", "test/cli/commands_test.py::test_encoding[utf-32-UTF-32]", "test/cli/commands_test.py::test_cli_encoding[utf-8-command-line-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-command-line-True]", "test/cli/commands_test.py::test_cli_encoding[utf-8-config-file-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-config-file-True]", "test/cli/commands_test.py::test_cli_no_disable_noqa_flag", "test/cli/commands_test.py::test_cli_disable_noqa_flag", "test/cli/commands_test.py::test_cli_warn_unused_noqa_flag", "test/cli/commands_test.py::test_cli_get_default_config", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar_deprecated_option", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_paths", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_files", "test/cli/commands_test.py::TestProgressBars::test_cli_fix_disabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_fix_disabled_progress_bar_deprecated_option", "test/cli/commands_test.py::test__cli__fix_multiple_errors_no_show_errors", "test/cli/commands_test.py::test__cli__fix_multiple_errors_quiet_force", "test/cli/commands_test.py::test__cli__fix_multiple_errors_quiet_no_force", "test/cli/commands_test.py::test__cli__fix_multiple_errors_show_errors", "test/cli/commands_test.py::test__cli__multiple_files__fix_multiple_errors_show_errors", "test/cli/commands_test.py::test__cli__render_fail", "test/cli/commands_test.py::test__cli__render_pass"], "environment_setup_commit": "7b7fd603a19755a9f3707ebbf95d18ee635716d8"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-3436", "base_commit": "23cd31e77a712a210c734e38488d7a34afd83a25", "patch": "diff --git a/src/sqlfluff/core/templaters/slicers/tracer.py b/src/sqlfluff/core/templaters/slicers/tracer.py\n--- a/src/sqlfluff/core/templaters/slicers/tracer.py\n+++ b/src/sqlfluff/core/templaters/slicers/tracer.py\n@@ -40,6 +40,7 @@ class RawSliceInfo:\n unique_alternate_id: Optional[str]\n alternate_code: Optional[str]\n next_slice_indices: List[int] = field(default_factory=list)\n+ inside_block: bool = field(default=False) # {% block %}\n \n \n class JinjaTracer:\n@@ -101,13 +102,21 @@ def trace(self, append_to_templated: str = \"\") -> JinjaTrace:\n alt_id, content_info, literal = value\n target_slice_idx = self.find_slice_index(alt_id)\n slice_length = content_info if literal else len(str(content_info))\n- self.move_to_slice(target_slice_idx, slice_length)\n+ target_inside_block = self.raw_slice_info[\n+ self.raw_sliced[target_slice_idx]\n+ ].inside_block\n+ if not target_inside_block:\n+ # Normal case: Walk through the template.\n+ self.move_to_slice(target_slice_idx, slice_length)\n+ else:\n+ # {% block %} executes code elsewhere in the template but does\n+ # not move there. It's a bit like macro invocation.\n+ self.record_trace(slice_length, target_slice_idx)\n \n # TRICKY: The 'append_to_templated' parameter is only used by the dbt\n # templater, passing \"\\n\" for this parameter if we need to add one back.\n # (The Jinja templater does not pass this parameter, so\n # 'append_to_templated' gets the default value of \"\", empty string.)\n- # we receive the default value of \"\".) The dbt templater will\n # For more detail, see the comments near the call to slice_file() in\n # plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py.\n templated_str = self.make_template(self.raw_str).render() + append_to_templated\n@@ -197,7 +206,8 @@ def __init__(self, raw_str: str, env: Environment):\n \n # Internal bookkeeping\n self.slice_id: int = 0\n- self.inside_set_or_macro: bool = False\n+ self.inside_set_or_macro: bool = False # {% set %} or {% macro %}\n+ self.inside_block = False # {% block %}\n self.stack: List[int] = []\n self.idx_raw: int = 0\n \n@@ -211,7 +221,7 @@ def slice_info_for_literal(self, length, prefix=\"\") -> RawSliceInfo:\n \"\"\"Returns a RawSliceInfo for a literal.\n \n In the alternate template, literals are replaced with a uniquely\n- numbered, easily-to-parse literal. JinjaTracer uses this output as\n+ numbered, easy-to-parse literal. JinjaTracer uses this output as\n a \"breadcrumb trail\" to deduce the execution path through the template.\n \n This is important even if the original literal (i.e. in the raw SQL\n@@ -222,13 +232,16 @@ def slice_info_for_literal(self, length, prefix=\"\") -> RawSliceInfo:\n \"\"\"\n unique_alternate_id = self.next_slice_id()\n alternate_code = f\"\\0{prefix}{unique_alternate_id}_{length}\"\n- return self.make_raw_slice_info(unique_alternate_id, alternate_code)\n+ return self.make_raw_slice_info(\n+ unique_alternate_id, alternate_code, inside_block=self.inside_block\n+ )\n \n- def update_inside_set_or_macro(\n+ def update_inside_set_or_macro_or_block(\n self, block_type: str, trimmed_parts: List[str]\n ) -> None:\n \"\"\"Based on block tag, update whether we're in a set/macro section.\"\"\"\n if block_type == \"block_start\" and trimmed_parts[0] in (\n+ \"block\",\n \"macro\",\n \"set\",\n ):\n@@ -236,11 +249,12 @@ def update_inside_set_or_macro(\n # - {% set variable = value %}\n # - {% set variable %}value{% endset %}\n # https://jinja.palletsprojects.com/en/2.10.x/templates/#block-assignments\n- # When the second format is used, set the field\n- # 'inside_set_or_macro' to True. This info is used elsewhere,\n- # as other code inside these regions require special handling.\n- # (Generally speaking, JinjaTracer ignores the contents of these\n- # blocks, treating them like opaque templated regions.)\n+ # When the second format is used, set one of the fields\n+ # 'inside_set_or_macro' or 'inside_block' to True. This info is\n+ # used elsewhere, as other code inside these regions require\n+ # special handling. (Generally speaking, JinjaAnalyzer ignores\n+ # the contents of these blocks, treating them like opaque templated\n+ # regions.)\n try:\n # Entering a set/macro block. Build a source string consisting\n # of just this one Jinja command and see if it parses. If so,\n@@ -255,22 +269,33 @@ def update_inside_set_or_macro(\n isinstance(e.message, str)\n and \"Unexpected end of template\" in e.message\n ):\n- # It was opening a block, thus we're inside a set or macro.\n- self.inside_set_or_macro = True\n+ # It was opening a block, thus we're inside a set, macro, or\n+ # block.\n+ if trimmed_parts[0] == \"block\":\n+ self.inside_block = True\n+ else:\n+ self.inside_set_or_macro = True\n else:\n raise # pragma: no cover\n- elif block_type == \"block_end\" and (trimmed_parts[0] in (\"endmacro\", \"endset\")):\n- # Exiting a set/macro block.\n- self.inside_set_or_macro = False\n+ elif block_type == \"block_end\":\n+ if trimmed_parts[0] in (\"endmacro\", \"endset\"):\n+ # Exiting a set or macro.\n+ self.inside_set_or_macro = False\n+ elif trimmed_parts[0] == \"endblock\":\n+ # Exiting a {% block %} block.\n+ self.inside_block = False\n \n def make_raw_slice_info(\n- self, unique_alternate_id: Optional[str], alternate_code: Optional[str]\n+ self,\n+ unique_alternate_id: Optional[str],\n+ alternate_code: Optional[str],\n+ inside_block: bool = False,\n ) -> RawSliceInfo:\n \"\"\"Create RawSliceInfo as given, or \"empty\" if in set/macro block.\"\"\"\n if not self.inside_set_or_macro:\n- return RawSliceInfo(unique_alternate_id, alternate_code, [])\n+ return RawSliceInfo(unique_alternate_id, alternate_code, [], inside_block)\n else:\n- return RawSliceInfo(None, None, [])\n+ return RawSliceInfo(None, None, [], False)\n \n # We decide the \"kind\" of element we're dealing with using its _closing_\n # tag rather than its opening tag. The types here map back to similar types\n@@ -351,7 +376,7 @@ def analyze(self, make_template: Callable[[str], Template]) -> JinjaTracer:\n raw_slice_info = self.track_templated(\n m_open, m_close, tag_contents\n )\n- self.update_inside_set_or_macro(block_type, tag_contents)\n+ self.update_inside_set_or_macro_or_block(block_type, tag_contents)\n m_strip_right = regex.search(\n r\"\\s+$\", raw, regex.MULTILINE | regex.DOTALL\n )\n", "test_patch": "diff --git a/test/core/templaters/jinja_test.py b/test/core/templaters/jinja_test.py\n--- a/test/core/templaters/jinja_test.py\n+++ b/test/core/templaters/jinja_test.py\n@@ -1060,6 +1060,61 @@ def test__templater_jinja_slice_template(test, result):\n (\"block_end\", slice(27, 39, None), slice(13, 13, None)),\n ],\n ),\n+ (\n+ # Test for issue 3434: Handle {% block %}.\n+ \"SELECT {% block table_name %}block_contents{% endblock %} \"\n+ \"FROM {{ self.table_name() }}\\n\",\n+ None,\n+ [\n+ (\"literal\", slice(0, 7, None), slice(0, 7, None)),\n+ (\"literal\", slice(29, 43, None), slice(7, 21, None)),\n+ (\"block_start\", slice(7, 29, None), slice(21, 21, None)),\n+ (\"literal\", slice(29, 43, None), slice(21, 21, None)),\n+ (\"block_end\", slice(43, 57, None), slice(21, 21, None)),\n+ (\"literal\", slice(57, 63, None), slice(21, 27, None)),\n+ (\"templated\", slice(63, 86, None), slice(27, 27, None)),\n+ (\"literal\", slice(29, 43, None), slice(27, 41, None)),\n+ (\"literal\", slice(86, 87, None), slice(41, 42, None)),\n+ ],\n+ ),\n+ (\n+ # Another test for issue 3434: Similar to the first, but uses\n+ # the block inside a loop.\n+ \"\"\"{% block table_name %}block_contents{% endblock %}\n+SELECT\n+{% for j in [4, 5, 6] %}\n+FROM {{ j }}{{ self.table_name() }}\n+{% endfor %}\n+\"\"\",\n+ None,\n+ [\n+ (\"literal\", slice(22, 36, None), slice(0, 14, None)),\n+ (\"block_start\", slice(0, 22, None), slice(14, 14, None)),\n+ (\"literal\", slice(22, 36, None), slice(14, 14, None)),\n+ (\"block_end\", slice(36, 50, None), slice(14, 14, None)),\n+ (\"literal\", slice(50, 58, None), slice(14, 22, None)),\n+ (\"block_start\", slice(58, 82, None), slice(22, 22, None)),\n+ (\"literal\", slice(82, 88, None), slice(22, 28, None)),\n+ (\"templated\", slice(88, 95, None), slice(28, 29, None)),\n+ (\"templated\", slice(95, 118, None), slice(29, 29, None)),\n+ (\"literal\", slice(22, 36, None), slice(29, 43, None)),\n+ (\"literal\", slice(118, 119, None), slice(43, 44, None)),\n+ (\"block_end\", slice(119, 131, None), slice(44, 44, None)),\n+ (\"literal\", slice(82, 88, None), slice(44, 50, None)),\n+ (\"templated\", slice(88, 95, None), slice(50, 51, None)),\n+ (\"templated\", slice(95, 118, None), slice(51, 51, None)),\n+ (\"literal\", slice(22, 36, None), slice(51, 65, None)),\n+ (\"literal\", slice(118, 119, None), slice(65, 66, None)),\n+ (\"block_end\", slice(119, 131, None), slice(66, 66, None)),\n+ (\"literal\", slice(82, 88, None), slice(66, 72, None)),\n+ (\"templated\", slice(88, 95, None), slice(72, 73, None)),\n+ (\"templated\", slice(95, 118, None), slice(73, 73, None)),\n+ (\"literal\", slice(22, 36, None), slice(73, 87, None)),\n+ (\"literal\", slice(118, 119, None), slice(87, 88, None)),\n+ (\"block_end\", slice(119, 131, None), slice(88, 88, None)),\n+ (\"literal\", slice(131, 132, None), slice(88, 89, None)),\n+ ],\n+ ),\n ],\n )\n def test__templater_jinja_slice_file(raw_file, override_context, result, caplog):\n", "problem_statement": "Fatal templating error with Jinja templater. Tracer produces odd results.\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nIssue found while assessing an Airflow project.\r\n\r\nThe smallest query I can make which triggers the issue is: \r\n```sql\r\nSELECT\r\n\t{% block table_name %}a{% endblock %}.b\r\nFROM d.{{ self.table_name() }}\r\n```\r\n\r\nWhen running this query through `lint` I get an `AssertionError`, or if running on the more friendly error message PR (#3433) I get: `WARNING Length of templated file mismatch with final slice: 21 != 19.`.\n\n### Expected Behaviour\n\nThis query should slice properly and probably eventually give a jinja error that the required variables are undefined.\n\n### Observed Behaviour\n\nI've dug a little into the error and the sliced file being produced is:\r\n\r\n```python\r\n[\r\n TemplatedFileSlice(slice_type='literal', source_slice=slice(0, 8, None), templated_slice=slice(0, 8, None)),\r\n TemplatedFileSlice(slice_type='block_start', source_slice=slice(8, 30, None), templated_slice=slice(8, 8, None)),\r\n TemplatedFileSlice(slice_type='literal', source_slice=slice(30, 31, None), templated_slice=slice(8, 9, None)),\r\n TemplatedFileSlice(slice_type='block_end', source_slice=slice(31, 45, None), templated_slice=slice(9, 9, None)),\r\n TemplatedFileSlice(slice_type='literal', source_slice=slice(45, 55, None), templated_slice=slice(9, 19, None)),\r\n TemplatedFileSlice(slice_type='templated', source_slice=slice(55, 78, None), templated_slice=slice(19, 19, None)),\r\n TemplatedFileSlice(slice_type='literal', source_slice=slice(78, 79, None), templated_slice=slice(19, 19, None))\r\n]\r\n```\r\n\r\nThe issue is that while the `source_slice` looks correct for the slices, almost all of the `templated_slices` values have zero length, and importantly the last one doesn't end at position 21.\r\n\r\nThe rendered file is `SELECT\\n\\ta.b\\nFROM d.a\\n` (I've included the escape chars) which is indeed 21 chars long.\r\n\r\n@barrywhart I might need your help to work out what's going on with the Jinja tracer here.\n\n### How to reproduce\n\nRun provided query, `main` branch. Set to the `jinja` templater.\n\n### Dialect\n\ndialect is set to `snowflake`, but I don't think we're getting far enough for that to make a difference.\n\n### Version\n\n`main` branch commit `cb6357c540d2d968f766f3a7a4fa16f231cb80e4` (and a few branches derived from it)\n\n### Configuration\n\nN/A\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [ ] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "I'll take a look.\r\n\r\nAnd darn it -- first bug report against this code in the past couple months, I think. \ud83d\ude05\nStarting to look at this. One problem I noticed (perhaps not the only one) is that the trailing literal newline in the source string has no corresponding templated slice, so it's like building the templated slice array has stopped early for some reason.\r\n\r\nThe 0-length slices may be legit. Will share more as I learn things, but is `{% block %}` a Jinja builtin or an extension? If it's an extension, maybe base Jinja is just skipping it (i.e. rendering it as empty string). \nOk, I think the issue is not related to undefined variables. I get the same assertion error if I define the variable prior to the block, e.g.:\r\n```\r\n{% set table_name = \"abc\" %}\r\nSELECT {% block table_name %}a{% endblock %} FROM {{ self.table_name() }}\r\n```\r\n\r\nI'm pretty sure the real issue is that we aren't handling `{% block %}` correctly **at all** (probably because I hadn't heard of it before \ud83e\udd2a).\r\n\r\nII think it should be handled similarly to `{% set %}` or `{% macro %}` blocks, i.e. basically don't trace when they are **defined**, only when they are **used**.\r\n\r\nI should be able to fix it this week. For now, just need to let my brain recover from looking at this code again. Even though I wrote it, it's a little too \"meta\" for me to stare at it for more than 1-2 hours at a time. \ud83d\ude05", "created_at": "2022-06-07T21:36:59Z", "version": "0.13", "FAIL_TO_PASS": ["test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%"], "PASS_TO_PASS": ["test/core/templaters/jinja_test.py::test__templater_jinja[simple]", "test/core/templaters/jinja_test.py::test__templater_jinja[unboundlocal_bugfix]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_left_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_right_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_comment]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[union_all_loop1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[set_multiple_variables_and_define_macro]", "test/core/templaters/jinja_test.py::test_templater_set_block_handling", "test/core/templaters/jinja_test.py::test__templater_jinja_error_variable", "test/core/templaters/jinja_test.py::test__templater_jinja_dynamic_variable_no_violations", "test/core/templaters/jinja_test.py::test__templater_jinja_error_syntax", "test/core/templaters/jinja_test.py::test__templater_jinja_error_catastrophic", "test/core/templaters/jinja_test.py::test__templater_jinja_error_macro_path_does_not_exist", "test/core/templaters/jinja_test.py::test__templater_jinja_lint_empty", "test/core/templaters/jinja_test.py::test__templater_full[jinja_a/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_b/jinja-False-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_config-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_is_incremental-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_ref-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_source-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_this-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_var_default-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_e/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_f/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_g_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_h_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag_2-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_j_libraries/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_k_config_override_path_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/001-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/002-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_m_libraries_module/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_n_nested_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_o_config_override_dbt_builtins/override_dbt_builtins-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_p_disable_dbt_builtins/disable_dbt_builtins-True-False]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[{%", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[-None-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[foo-None-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{{", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT\\n", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%-", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[select\\n"], "environment_setup_commit": "6e8ce43a4958dbaa56256365c2a89d8db92e07d6"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-2849", "base_commit": "0bbd70f38a3318b9a488d988d06e8005e222d6ac", "patch": "diff --git a/plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py b/plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py\n--- a/plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py\n+++ b/plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py\n@@ -522,17 +522,21 @@ def make_template(in_str):\n # sliced_file to reflect the mapping of the added character(s) back\n # to the raw SQL.\n templated_sql = templated_sql + \"\\n\" * n_trailing_newlines\n- sliced_file.append(\n- TemplatedFileSlice(\n- slice_type=\"literal\",\n- source_slice=slice(\n- len(source_dbt_sql) - n_trailing_newlines, len(source_dbt_sql)\n- ),\n- templated_slice=slice(\n- len(templated_sql) - n_trailing_newlines, len(templated_sql)\n- ),\n+ if sliced_file and sliced_file[-1].templated_slice.stop != len(\n+ templated_sql\n+ ):\n+ sliced_file.append(\n+ TemplatedFileSlice(\n+ slice_type=\"literal\",\n+ source_slice=slice(\n+ len(source_dbt_sql) - n_trailing_newlines,\n+ len(source_dbt_sql),\n+ ),\n+ templated_slice=slice(\n+ len(templated_sql) - n_trailing_newlines, len(templated_sql)\n+ ),\n+ )\n )\n- )\n return (\n TemplatedFile(\n source_str=source_dbt_sql,\ndiff --git a/src/sqlfluff/core/templaters/base.py b/src/sqlfluff/core/templaters/base.py\n--- a/src/sqlfluff/core/templaters/base.py\n+++ b/src/sqlfluff/core/templaters/base.py\n@@ -73,6 +73,7 @@ def __init__(\n templated_str: Optional[str] = None,\n sliced_file: Optional[List[TemplatedFileSlice]] = None,\n raw_sliced: Optional[List[RawFileSlice]] = None,\n+ check_consistency=True,\n ):\n \"\"\"Initialise the TemplatedFile.\n \n@@ -104,6 +105,36 @@ def __init__(\n self._source_newlines = list(iter_indices_of_newlines(self.source_str))\n self._templated_newlines = list(iter_indices_of_newlines(self.templated_str))\n \n+ # NOTE: The \"check_consistency\" flag should always be True when using\n+ # SQLFluff in real life. This flag was only added because some legacy\n+ # templater tests in test/core/templaters/jinja_test.py use hardcoded\n+ # test data with issues that will trigger errors here. It would be cool\n+ # to fix that data someday. I (Barry H.) started looking into it, but\n+ # it was much trickier than I expected, because bits of the same data\n+ # are shared across multiple tests.\n+ if check_consistency:\n+ # Sanity check raw string and slices.\n+ pos = 0\n+ rfs: RawFileSlice\n+ for idx, rfs in enumerate(self.raw_sliced):\n+ assert rfs.source_idx == pos\n+ pos += len(rfs.raw)\n+ assert pos == len(self.source_str)\n+\n+ # Sanity check templated string and slices.\n+ previous_slice = None\n+ tfs: Optional[TemplatedFileSlice] = None\n+ for idx, tfs in enumerate(self.sliced_file):\n+ if previous_slice:\n+ assert (\n+ tfs.templated_slice.start == previous_slice.templated_slice.stop\n+ )\n+ else:\n+ assert tfs.templated_slice.start == 0\n+ previous_slice = tfs\n+ if self.sliced_file and templated_str is not None:\n+ assert tfs.templated_slice.stop == len(templated_str)\n+\n @classmethod\n def from_string(cls, raw):\n \"\"\"Create TemplatedFile from a string.\"\"\"\ndiff --git a/src/sqlfluff/core/templaters/slicers/tracer.py b/src/sqlfluff/core/templaters/slicers/tracer.py\n--- a/src/sqlfluff/core/templaters/slicers/tracer.py\n+++ b/src/sqlfluff/core/templaters/slicers/tracer.py\n@@ -77,9 +77,6 @@ def trace(self) -> JinjaTrace:\n except IndexError:\n pos2 = len(trace_template_output)\n p = trace_template_output[pos1 + 1 : pos2]\n- is_set_or_macro = p[:3] == \"set\"\n- if is_set_or_macro:\n- p = p[3:]\n m_id = regex.match(r\"^([0-9a-f]+)(_(\\d+))?\", p)\n if not m_id:\n raise ValueError( # pragma: no cover\n@@ -98,18 +95,7 @@ def trace(self) -> JinjaTrace:\n alt_id, content_info, literal = value\n target_slice_idx = self.find_slice_index(alt_id)\n slice_length = content_info if literal else len(str(content_info))\n- if not is_set_or_macro:\n- self.move_to_slice(target_slice_idx, slice_length)\n- else:\n- # If we find output from a {% set %} directive or a macro,\n- # record a trace without reading or updating the program\n- # counter. Such slices are always treated as \"templated\"\n- # because they are inserted during expansion of templated\n- # code (i.e. {% set %} variable or macro defined within the\n- # file).\n- self.record_trace(\n- slice_length, target_slice_idx, slice_type=\"templated\"\n- )\n+ self.move_to_slice(target_slice_idx, slice_length)\n return JinjaTrace(\n self.make_template(self.raw_str).render(), self.raw_sliced, self.sliced_file\n )\n@@ -241,9 +227,17 @@ def _slice_template(self) -> List[RawFileSlice]:\n idx,\n )\n )\n- self.raw_slice_info[result[-1]] = self.slice_info_for_literal(\n- len(raw), \"\" if set_idx is None else \"set\"\n- )\n+ if set_idx is None:\n+ rsi = self.slice_info_for_literal(\n+ len(raw), \"\" if set_idx is None else \"set\"\n+ )\n+ else:\n+ # For \"set\" blocks, don't generate alternate ID or code.\n+ # Sometimes, dbt users use {% set %} blocks to generate\n+ # queries that get sent to actual databases, thus causing\n+ # errors if we tamper with it.\n+ rsi = RawSliceInfo(None, None, [])\n+ self.raw_slice_info[result[-1]] = rsi\n idx += len(raw)\n continue\n str_buff += raw\n@@ -326,15 +320,20 @@ def _slice_template(self) -> List[RawFileSlice]:\n # effects, but return a unique slice ID.\n if trimmed_content:\n assert m_open and m_close\n- unique_id = self.next_slice_id()\n- unique_alternate_id = unique_id\n- prefix = \"set\" if set_idx is not None else \"\"\n- open_ = m_open.group(1)\n- close_ = m_close.group(1)\n- alternate_code = (\n- f\"\\0{prefix}{unique_alternate_id} {open_} \"\n- f\"{trimmed_content} {close_}\"\n- )\n+ # For \"set\" blocks, don't generate alternate ID or\n+ # code. Sometimes, dbt users use {% set %} blocks to\n+ # generate queries that get sent to actual\n+ # databases, thus causing errors if we tamper with\n+ # it.\n+ if set_idx is None:\n+ unique_id = self.next_slice_id()\n+ unique_alternate_id = unique_id\n+ open_ = m_open.group(1)\n+ close_ = m_close.group(1)\n+ alternate_code = (\n+ f\"\\0{unique_alternate_id} {open_} \"\n+ f\"{trimmed_content} {close_}\"\n+ )\n if block_type == \"block_start\" and trimmed_content.split()[0] in (\n \"macro\",\n \"set\",\n@@ -343,16 +342,24 @@ def _slice_template(self) -> List[RawFileSlice]:\n # - {% set variable = value %}\n # - {% set variable %}value{% endset %}\n # https://jinja.palletsprojects.com/en/2.10.x/templates/#block-assignments\n- # When the second format is used, set the variable 'is_set'\n+ # When the second format is used, set the variable 'set_idx'\n # to a non-None value. This info is used elsewhere, as\n # literals inside a {% set %} block require special handling\n # during the trace.\n trimmed_content_parts = trimmed_content.split(maxsplit=2)\n- if len(trimmed_content_parts) <= 2 or not trimmed_content_parts[\n- 2\n- ].startswith(\"=\"):\n+ if len(trimmed_content_parts) <= 2 or (\n+ not trimmed_content_parts[1].endswith(\"=\")\n+ and not trimmed_content_parts[2].startswith(\"=\")\n+ ):\n set_idx = len(result)\n- elif block_type == \"block_end\" and set_idx is not None:\n+ elif (\n+ block_type == \"block_end\"\n+ and set_idx is not None\n+ and (\n+ trimmed_content.startswith(\"endset\")\n+ or trimmed_content.startswith(\"endmacro\")\n+ )\n+ ):\n # Exiting a {% set %} block. Clear the indicator variable.\n set_idx = None\n m = regex.search(r\"\\s+$\", raw, regex.MULTILINE | regex.DOTALL)\n", "test_patch": "diff --git a/test/core/templaters/base_test.py b/test/core/templaters/base_test.py\n--- a/test/core/templaters/base_test.py\n+++ b/test/core/templaters/base_test.py\n@@ -134,6 +134,7 @@ def test__templated_file_get_line_pos_of_char_pos(\n templated_str=templated_str,\n sliced_file=file_slices,\n fname=\"test\",\n+ check_consistency=False,\n )\n res_line_no, res_line_pos = file.get_line_pos_of_char_pos(in_charpos)\n assert res_line_no == out_line_no\n@@ -287,6 +288,7 @@ def test__templated_file_templated_slice_to_source_slice(\n for rs in raw_slices\n ],\n fname=\"test\",\n+ check_consistency=False,\n )\n source_slice = file.templated_slice_to_source_slice(in_slice)\n literal_test = file.is_source_slice_literal(source_slice)\n@@ -303,5 +305,6 @@ def test__templated_file_source_only_slices():\n RawFileSlice(\"b\" * 7, \"comment\", 10),\n RawFileSlice(\"a\" * 10, \"literal\", 17),\n ],\n+ check_consistency=False,\n )\n assert file.source_only_slices() == [RawFileSlice(\"b\" * 7, \"comment\", 10)]\ndiff --git a/test/core/templaters/jinja_test.py b/test/core/templaters/jinja_test.py\n--- a/test/core/templaters/jinja_test.py\n+++ b/test/core/templaters/jinja_test.py\n@@ -370,6 +370,35 @@ def test__templater_jinja_slices(case: RawTemplatedTestCase):\n assert actual_rs_source_list == case.expected_raw_sliced__source_list\n \n \n+def test_templater_set_block_handling():\n+ \"\"\"Test handling of literals in {% set %} blocks.\n+\n+ Specifically, verify they are not modified in the alternate template.\n+ \"\"\"\n+\n+ def run_query(sql):\n+ # Prior to the bug fix, this assertion failed. This was bad because,\n+ # inside JinjaTracer, dbt templates similar to the one in this test\n+ # would call the database with funky SQL (including weird strings it\n+ # uses internally like: 00000000000000000000000000000002.\n+ assert sql == \"\\n\\nselect 1 from foobarfoobarfoobarfoobar_dev\\n\\n\"\n+ return sql\n+\n+ t = JinjaTemplater(override_context=dict(run_query=run_query))\n+ instr = \"\"\"{% set my_query1 %}\n+select 1 from foobarfoobarfoobarfoobar_{{ \"dev\" }}\n+{% endset %}\n+{% set my_query2 %}\n+{{ my_query1 }}\n+{% endset %}\n+\n+{{ run_query(my_query2) }}\n+\"\"\"\n+ outstr, vs = t.process(in_str=instr, fname=\"test\", config=FluffConfig())\n+ assert str(outstr) == \"\\n\\n\\n\\n\\nselect 1 from foobarfoobarfoobarfoobar_dev\\n\\n\\n\"\n+ assert len(vs) == 0\n+\n+\n def test__templater_jinja_error_variable():\n \"\"\"Test missing variable error handling in the jinja templater.\"\"\"\n t = JinjaTemplater(override_context=dict(blah=\"foo\"))\n@@ -846,6 +875,45 @@ def test__templater_jinja_slice_template(test, result):\n (\"literal\", slice(312, 327, None), slice(27, 42, None)),\n ],\n ),\n+ (\n+ # Test for issue 2835. There's no space between \"col\" and \"=\"\n+ \"\"\"{% set col= \"col1\" %}\n+SELECT {{ col }}\n+\"\"\",\n+ None,\n+ [\n+ (\"block_start\", slice(0, 21, None), slice(0, 0, None)),\n+ (\"literal\", slice(21, 29, None), slice(0, 8, None)),\n+ (\"templated\", slice(29, 38, None), slice(8, 12, None)),\n+ (\"literal\", slice(38, 39, None), slice(12, 13, None)),\n+ ],\n+ ),\n+ (\n+ # Another test for issue 2835. The {% for %} loop inside the\n+ # {% set %} caused JinjaTracer to think the {% set %} ended\n+ # at the {% endfor %}\n+ \"\"\"{% set some_part_of_the_query %}\n+ {% for col in [\"col1\"] %}\n+ {{col}}\n+ {% endfor %}\n+{% endset %}\n+\n+SELECT {{some_part_of_the_query}}\n+FROM SOME_TABLE\n+\"\"\",\n+ None,\n+ [\n+ (\"block_start\", slice(0, 32, None), slice(0, 0, None)),\n+ (\"literal\", slice(32, 37, None), slice(0, 0, None)),\n+ (\"block_start\", slice(37, 62, None), slice(0, 0, None)),\n+ (\"block_end\", slice(79, 91, None), slice(0, 0, None)),\n+ (\"literal\", slice(91, 92, None), slice(0, 0, None)),\n+ (\"block_end\", slice(92, 104, None), slice(0, 0, None)),\n+ (\"literal\", slice(104, 113, None), slice(0, 9, None)),\n+ (\"templated\", slice(113, 139, None), slice(9, 29, None)),\n+ (\"literal\", slice(139, 156, None), slice(29, 46, None)),\n+ ],\n+ ),\n ],\n )\n def test__templater_jinja_slice_file(raw_file, override_context, result, caplog):\n", "problem_statement": "Lint and fix throws exception when having jinja for loop inside set\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nTo reproduce the error, create test.template.sql\r\n```\r\n{% set whitelisted= [\r\n {'name': 'COL_1'},\r\n {'name': 'COL_2'},\r\n {'name': 'COL_3'}\r\n] %}\r\n\r\n{% set some_part_of_the_query %}\r\n {% for col in whitelisted %}\r\n {{col.name}}{{ \", \" if not loop.last }}\r\n {% endfor %}\r\n{% endset %}\r\n\r\nSELECT {{some_part_of_the_query}}\r\nFROM SOME_TABLE\r\n\r\n```\r\n\r\nwhen running lint i get this error:\r\n```\r\n==== sqlfluff ====\r\nsqlfluff: 0.11.0 python: 3.8.12\r\nimplementation: cpython dialect: snowflake\r\nverbosity: 1 templater: jinja\r\n\r\n==== readout ====\r\n\r\n=== [ path: test.template.sql ] ===\r\n\r\nWARNING Unable to lint test.template.sql due to an internal error. Please report this as an issue with your query's contents and stacktrace below!\r\nTo hide this warning, add the failing file to .sqlfluffignore\r\nTraceback (most recent call last):\r\n File \"lib/python3.8/site-packages/sqlfluff/core/linter/runner.py\", line 103, in run\r\n yield partial()\r\n File \"lib/python3.8/site-packages/sqlfluff/core/linter/linter.py\", line 666, in lint_rendered\r\n parsed = cls.parse_rendered(rendered)\r\n File \"lib/python3.8/site-packages/sqlfluff/core/linter/linter.py\", line 352, in parse_rendered\r\n tokens, lvs, config = cls._lex_templated_file(\r\n File \"lib/python3.8/site-packages/sqlfluff/core/linter/linter.py\", line 139, in _lex_templated_file\r\n tokens, lex_vs = lexer.lex(templated_file)\r\n File \"lib/python3.8/site-packages/sqlfluff/core/parser/lexer.py\", line 321, in lex\r\n segments: Tuple[RawSegment, ...] = self.elements_to_segments(\r\n File \"lib/python3.8/site-packages/sqlfluff/core/parser/lexer.py\", line 348, in elements_to_segments\r\n source_slice = templated_file.templated_slice_to_source_slice(\r\n File \"lib/python3.8/site-packages/sqlfluff/core/templaters/base.py\", line 258, in templated_slice_to_source_slice\r\n ts_stop_sf_start, ts_stop_sf_stop = self._find_slice_indices_of_templated_pos(\r\n File \"lib/python3.8/site-packages/sqlfluff/core/templaters/base.py\", line 177, in _find_slice_indices_of_templated_pos\r\n raise ValueError(\"Position Not Found\")\r\nValueError: Position Not Found\r\n \r\n==== summary ====\r\nviolations: 0 status: PASS\r\nAll Finished \ud83d\udcdc \ud83c\udf89!\r\n\r\n```\r\n\r\nThis is the rendered query:\r\n```\r\n SELECT\r\n\r\n COL_1,\r\n\r\n COL_2,\r\n\r\n COL_3\r\n\r\n\r\nFROM SOME_TABLE\r\n\r\n```\r\n\r\nAnd when trying around to make this work i removed the new lines between the selected columns like this:\r\n```\r\n{% set whitelisted= [\r\n {'name': 'COL_1'},\r\n {'name': 'COL_2'},\r\n {'name': 'COL_3'}\r\n] %}\r\n\r\n{% set some_part_of_the_query %}\r\n {% for col in whitelisted -%}\r\n {{col.name}}{{ \", \" if not loop.last }}\r\n {% endfor -%}\r\n{% endset %}\r\n\r\nSELECT {{some_part_of_the_query}}\r\nFROM SOME_TABLE\r\n\r\n```\r\n\r\nwhich renders:\r\n```\r\nSELECT\r\n COL_1,\r\n COL_2,\r\n COL_3\r\n\r\nFROM SOME_TABLE\r\n\r\n```\r\n\r\nAnd this will make the linter pass:\r\n\r\n```\r\n==== sqlfluff ====\r\nsqlfluff: 0.11.0 python: 3.8.12\r\nimplementation: cpython dialect: snowflake\r\nverbosity: 1 templater: jinja\r\n\r\n==== readout ====\r\n\r\n=== [ path: test.template.sql ] ===\r\n\r\n== [test.template.sql] PASS \r\n==== summary ====\r\nviolations: 0 status: PASS\r\nAll Finished \ud83d\udcdc \ud83c\udf89!\r\n\r\n```\r\n\r\n\n\n### Expected Behaviour\n\nMy expectations is that the linter and fix should pass.\n\n### Observed Behaviour\n\nRight now lint and fix throws exception (see \"What Happened\" section)\n\n### How to reproduce\n\nMentioned above.\n\n### Dialect\n\nsnowflake\n\n### Version\n\nsqlfluff, version 0.11.0\n\n### Configuration\n\n[sqlfluff]\r\nverbose = 1\r\ndialect = snowflake\r\ntemplater = jinja\r\nexclude_rules = L027,L031,L032,L036,L044,L046,L034,L050\r\noutput_line_length = 121\r\nsql_file_exts=.sql\r\n\r\n[sqlfluff:rules]\r\ntab_space_size = 4\r\nmax_line_length = 250\r\nindent_unit = space\r\ncomma_style = trailing\r\nallow_scalar = True\r\nsingle_table_references = consistent\r\nunquoted_identifiers_policy = aliases\r\n\r\n[sqlfluff:rules:L042]\r\nforbid_subquery_in = both\r\n\r\n[sqlfluff:rules:L010] # Keywords\r\ncapitalisation_policy = upper\r\n\r\n[sqlfluff:rules:L014]\r\nextended_capitalisation_policy = lower\r\n\r\n[sqlfluff:rules:L030] # function names\r\nextended_capitalisation_policy = upper\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [ ] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "", "created_at": "2022-03-12T21:48:15Z", "version": "0.10", "FAIL_TO_PASS": ["test/core/templaters/base_test.py::test__templated_file_get_line_pos_of_char_pos[01234\\n6789{{foo}}fo\\nbarss-01234\\n6789x\\nfo\\nbarfss-file_slices0-0-1-1]", "test/core/templaters/base_test.py::test__templated_file_get_line_pos_of_char_pos[01234\\n6789{{foo}}fo\\nbarss-01234\\n6789x\\nfo\\nbarfss-file_slices1-20-3-1]", "test/core/templaters/base_test.py::test__templated_file_get_line_pos_of_char_pos[01234\\n6789{{foo}}fo\\nbarss-01234\\n6789x\\nfo\\nbarfss-file_slices2-24-3-5]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice0-out_slice0-True-file_slices0-raw_slices0]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice1-out_slice1-True-file_slices1-raw_slices1]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice2-out_slice2-True-file_slices2-raw_slices2]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice3-out_slice3-False-file_slices3-raw_slices3]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice4-out_slice4-False-file_slices4-raw_slices4]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice5-out_slice5-True-file_slices5-raw_slices5]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice6-out_slice6-True-file_slices6-raw_slices6]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice7-out_slice7-True-file_slices7-raw_slices7]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice8-out_slice8-True-file_slices8-raw_slices8]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice9-out_slice9-True-file_slices9-raw_slices9]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice10-out_slice10-True-file_slices10-raw_slices10]", "test/core/templaters/base_test.py::test__templated_file_templated_slice_to_source_slice[in_slice11-out_slice11-False-file_slices11-raw_slices11]", "test/core/templaters/base_test.py::test__templated_file_source_only_slices", "test/core/templaters/jinja_test.py::test_templater_set_block_handling", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%"], "PASS_TO_PASS": ["test/core/templaters/base_test.py::test__indices_of_newlines[-positions0]", "test/core/templaters/base_test.py::test__indices_of_newlines[foo-positions1]", "test/core/templaters/base_test.py::test__indices_of_newlines[foo\\nbar-positions2]", "test/core/templaters/base_test.py::test__indices_of_newlines[\\nfoo\\n\\nbar\\nfoo\\n\\nbar\\n-positions3]", "test/core/templaters/base_test.py::test__templater_raw", "test/core/templaters/base_test.py::test__templated_file_find_slice_indices_of_templated_pos[100-True-file_slices0-10-11]", "test/core/templaters/base_test.py::test__templated_file_find_slice_indices_of_templated_pos[13-True-file_slices1-0-3]", "test/core/templaters/base_test.py::test__templated_file_find_slice_indices_of_templated_pos[28-True-file_slices2-2-5]", "test/core/templaters/base_test.py::test__templated_file_find_slice_indices_of_templated_pos[12-True-file_slices3-1-3]", "test/core/templaters/base_test.py::test__templated_file_find_slice_indices_of_templated_pos[20-True-file_slices4-2-3]", "test/core/templaters/base_test.py::test__templated_file_find_slice_indices_of_templated_pos[13-False-file_slices5-0-1]", "test/core/templaters/jinja_test.py::test__templater_jinja[simple]", "test/core/templaters/jinja_test.py::test__templater_jinja[unboundlocal_bugfix]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_left_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_right_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_comment]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[union_all_loop1]", "test/core/templaters/jinja_test.py::test__templater_jinja_error_variable", "test/core/templaters/jinja_test.py::test__templater_jinja_error_syntax", "test/core/templaters/jinja_test.py::test__templater_jinja_error_catastrophic", "test/core/templaters/jinja_test.py::test__templater_jinja_error_macro_path_does_not_exist", "test/core/templaters/jinja_test.py::test__templater_jinja_lint_empty", "test/core/templaters/jinja_test.py::test__templater_full[jinja_a/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_b/jinja-False-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/var_default-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_e/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_f/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_g_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_h_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag_2-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_j_libraries/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_k_config_override_path_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/001-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/002-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_m_libraries_module/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_n_nested_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[{%", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[-None-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[foo-None-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{{", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT\\n", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%-", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[select\\n"], "environment_setup_commit": "3d52e8270d82aeccf4c516d059a80a6947919aea"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-884", "base_commit": "c0bad78f3fa9549591738c77f869724f721e6830", "patch": "diff --git a/src/sqlfluff/core/dialects/dialect_ansi.py b/src/sqlfluff/core/dialects/dialect_ansi.py\n--- a/src/sqlfluff/core/dialects/dialect_ansi.py\n+++ b/src/sqlfluff/core/dialects/dialect_ansi.py\n@@ -50,7 +50,7 @@\n ansi_dialect.set_lexer_struct(\n [\n # name, type, pattern, kwargs\n- (\"whitespace\", \"regex\", r\"[\\t ]+\", dict(type=\"whitespace\")),\n+ (\"whitespace\", \"regex\", r\"[\\t ]+\", dict(type=\"whitespace\", is_whitespace=True)),\n (\n \"inline_comment\",\n \"regex\",\n@@ -64,9 +64,14 @@\n dict(\n is_comment=True,\n type=\"comment\",\n- subdivide=dict(type=\"newline\", name=\"newline\", regex=r\"\\r\\n|\\n\"),\n+ subdivide=dict(\n+ type=\"newline\", name=\"newline\", regex=r\"\\r\\n|\\n\", is_whitespace=True\n+ ),\n trim_post_subdivide=dict(\n- type=\"whitespace\", name=\"whitespace\", regex=r\"[\\t ]+\"\n+ type=\"whitespace\",\n+ name=\"whitespace\",\n+ regex=r\"[\\t ]+\",\n+ is_whitespace=True,\n ),\n ),\n ),\n@@ -83,7 +88,7 @@\n (\"not_equal\", \"regex\", r\"!=|<>\", dict(is_code=True)),\n (\"greater_than_or_equal\", \"regex\", r\">=\", dict(is_code=True)),\n (\"less_than_or_equal\", \"regex\", r\"<=\", dict(is_code=True)),\n- (\"newline\", \"regex\", r\"\\r\\n|\\n\", dict(type=\"newline\")),\n+ (\"newline\", \"regex\", r\"\\r\\n|\\n\", dict(type=\"newline\", is_whitespace=True)),\n (\"casting_operator\", \"regex\", r\"::\", dict(is_code=True)),\n (\"concat_operator\", \"regex\", r\"\\|\\|\", dict(is_code=True)),\n (\"equals\", \"singleton\", \"=\", dict(is_code=True)),\ndiff --git a/src/sqlfluff/core/parser/lexer.py b/src/sqlfluff/core/parser/lexer.py\n--- a/src/sqlfluff/core/parser/lexer.py\n+++ b/src/sqlfluff/core/parser/lexer.py\n@@ -74,12 +74,10 @@ def _trim(self, matched, start_pos):\n idx = 0\n \n if self.trim_post_subdivide:\n- trimmer = re.compile(self.trim_post_subdivide[\"regex\"], re.DOTALL)\n- TrimClass = RawSegment.make(\n- self.trim_post_subdivide[\"regex\"],\n- name=self.trim_post_subdivide[\"name\"],\n- type=self.trim_post_subdivide[\"type\"],\n- )\n+ class_kwargs = self.trim_post_subdivide.copy()\n+ pattern = class_kwargs.pop(\"regex\")\n+ trimmer = re.compile(pattern, re.DOTALL)\n+ TrimClass = RawSegment.make(pattern, **class_kwargs)\n \n for trim_mat in trimmer.finditer(matched):\n trim_span = trim_mat.span()\n@@ -132,12 +130,10 @@ def _subdivide(self, matched, start_pos):\n seg_buff = ()\n str_buff = matched\n pos_buff = start_pos\n- divider = re.compile(self.subdivide[\"regex\"], re.DOTALL)\n- DividerClass = RawSegment.make(\n- self.subdivide[\"regex\"],\n- name=self.subdivide[\"name\"],\n- type=self.subdivide[\"type\"],\n- )\n+ class_kwargs = self.subdivide.copy()\n+ pattern = class_kwargs.pop(\"regex\")\n+ divider = re.compile(pattern, re.DOTALL)\n+ DividerClass = RawSegment.make(pattern, **class_kwargs)\n \n while True:\n # Iterate through subdividing as appropriate\n", "test_patch": "diff --git a/test/core/dialects/ansi_test.py b/test/core/dialects/ansi_test.py\n--- a/test/core/dialects/ansi_test.py\n+++ b/test/core/dialects/ansi_test.py\n@@ -162,3 +162,14 @@ def test__dialect__ansi_specific_segment_not_parse(raw, err_locations, caplog):\n assert len(parsed.violations) > 0\n locs = [(v.line_no(), v.line_pos()) for v in parsed.violations]\n assert locs == err_locations\n+\n+\n+def test__dialect__ansi_is_whitespace():\n+ \"\"\"Test proper tagging with is_whitespace.\"\"\"\n+ lnt = Linter()\n+ with open(\"test/fixtures/parser/ansi/select_in_multiline_comment.sql\") as f:\n+ parsed = lnt.parse_string(f.read())\n+ # Check all the segments that *should* be whitespace, ARE\n+ for raw_seg in parsed.tree.iter_raw_seg():\n+ if raw_seg.type in (\"whitespace\", \"newline\"):\n+ assert raw_seg.is_whitespace\n", "problem_statement": "Whitespace token is_whitespace is False\nI expect segment.is_whitespace of a Whitespace token is True, however, it is set to False.\r\n\r\n## Expected Behaviour\r\nsegment.is_whitespace return True\r\n\r\n## Observed Behaviour\r\nsegment.is_whitespace return False\r\n## Steps to Reproduce\r\n\r\n## Version\r\nInclude the output of `sqlfluff --version` along with your Python version\r\n\r\n## Configuration\r\n```\r\nInclude your SQLFluff configuration here\r\n```\r\n\n", "hints_text": "To triage this issue, I searched the SQLFluff code to find all uses of `is_whitespace`. This is the only one I found:\r\n```\r\nsrc/sqlfluff/core/parser/segments/base.py:72: is_whitespace = False\r\n```\r\n\r\n@alanmcruickshank: What's the purpose of `is_whitespace`?\r\n\r\nI see that long ago (2019), there was a class `WhitespaceSegment` (also a `NewlineSegment`). Now it's not a class -- instead, it'd defined in `src/sqlfluff/core/rules/base.py`.\nOnce #866 is merged I'll pick up the rest of this which relates to some of the lexer objects.", "created_at": "2021-03-23T21:28:49Z", "version": "0.4", "FAIL_TO_PASS": ["test/core/dialects/ansi_test.py::test__dialect__ansi_is_whitespace"], "PASS_TO_PASS": ["test/core/dialects/ansi_test.py::test__dialect__ansi__file_lex[a", "test/core/dialects/ansi_test.py::test__dialect__ansi__file_lex[b.c-res1]", "test/core/dialects/ansi_test.py::test__dialect__ansi__file_lex[abc", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectKeywordSegment-select]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[NakedIdentifierSegment-online_sales]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[BareFunctionSegment-current_timestamp]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[FunctionSegment-current_timestamp()]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[NumericLiteralSegment-1000.0]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-online_sales", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[IntervalExpressionSegment-INTERVAL", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-CASE", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-CAST(ROUND(online_sales", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-name", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectTargetElementSegment-MIN", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-DATE_ADD(CURRENT_DATE('America/New_York'),", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-my_array[1]]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-my_array[OFFSET(1)]]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-my_array[5:8]]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-4", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-bits[OFFSET(0)]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectTargetElementSegment-(count_18_24", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-count_18_24", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectStatementSegment-SELECT", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectTargetElementSegment-t.val/t.id]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectTargetElementSegment-CAST(num", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectTargetElementSegment-a.*]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectTargetElementSegment-a.b.*]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectTargetElementSegment-a.b.c.*]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ObjectReferenceSegment-a..c.*]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectTargetElementSegment--some_variable]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectTargetElementSegment--", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-concat(left(uaid,", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-c", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[ExpressionSegment-NULL::INT]", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_parses[SelectTargetElementSegment-NULL::INT", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_not_match[ObjectReferenceSegment-\\n", "test/core/dialects/ansi_test.py::test__dialect__ansi_specific_segment_not_parse[SELECT"], "environment_setup_commit": "cbdcfb09feb4883de91de142956c3be6ac7f827d"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-4151", "base_commit": "dc59c2a5672aacedaf91f0e6129b467eefad331b", "patch": "diff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py\n--- a/src/sqlfluff/cli/commands.py\n+++ b/src/sqlfluff/cli/commands.py\n@@ -680,7 +680,7 @@ def do_fixes(lnt, result, formatter=None, **kwargs):\n return False # pragma: no cover\n \n \n-@cli.command()\n+@cli.command(cls=DeprecatedOptionsCommand)\n @common_options\n @core_options\n @click.option(\n@@ -710,9 +710,12 @@ def do_fixes(lnt, result, formatter=None, **kwargs):\n ),\n )\n @click.option(\n+ \"--disable_progress_bar\",\n \"--disable-progress-bar\",\n is_flag=True,\n help=\"Disables progress bars.\",\n+ cls=DeprecatedOption,\n+ deprecated=[\"--disable_progress_bar\"],\n )\n @click.option(\n \"--FIX-EVEN-UNPARSABLE\",\n", "test_patch": "diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py\n--- a/test/cli/commands_test.py\n+++ b/test/cli/commands_test.py\n@@ -1775,6 +1775,46 @@ def test_cli_lint_enabled_progress_bar_multiple_files(\n assert r\"\\rrule L001:\" in raw_output\n assert r\"\\rrule L049:\" in raw_output\n \n+ def test_cli_fix_disabled_progress_bar(\n+ self, mock_disable_progress_bar: MagicMock\n+ ) -> None:\n+ \"\"\"When progress bar is disabled, nothing should be printed into output.\"\"\"\n+ result = invoke_assert_code(\n+ args=[\n+ fix,\n+ [\n+ \"--disable-progress-bar\",\n+ \"test/fixtures/linter/passing.sql\",\n+ ],\n+ ],\n+ )\n+ raw_output = repr(result.output)\n+\n+ assert (\n+ \"DeprecationWarning: The option '--disable_progress_bar' is deprecated, \"\n+ \"use '--disable-progress-bar'\"\n+ ) not in raw_output\n+\n+ def test_cli_fix_disabled_progress_bar_deprecated_option(\n+ self, mock_disable_progress_bar: MagicMock\n+ ) -> None:\n+ \"\"\"Same as above but checks additionally if deprecation warning is printed.\"\"\"\n+ result = invoke_assert_code(\n+ args=[\n+ fix,\n+ [\n+ \"--disable_progress_bar\",\n+ \"test/fixtures/linter/passing.sql\",\n+ ],\n+ ],\n+ )\n+ raw_output = repr(result.output)\n+\n+ assert (\n+ \"DeprecationWarning: The option '--disable_progress_bar' is deprecated, \"\n+ \"use '--disable-progress-bar'\"\n+ ) in raw_output\n+\n \n multiple_expected_output = \"\"\"==== finding fixable violations ====\n == [test/fixtures/linter/multiple_sql_errors.sql] FAIL\n", "problem_statement": "--disable_progress_bar Flag Broken for Fix\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nI ran `sqlfluff fix ${target} --dialect ansi --disable_progress_bar --force` on version 1.4.0 and got an error with exit code 2. Running with `--disable-progress-bar` appears to work fine, but it appears that compatibility with underscores was broken in version 1.4.0.\n\n### Expected Behaviour\n\nShould run as expected, with no error and no progress bar.\n\n### Observed Behaviour\n\nExit code 2 and stderr:\r\n```\r\nUsage: sqlfluff fix [OPTIONS] [PATHS]...\r\n Try 'sqlfluff fix -h' for help.\r\n\r\n Error: No such option: --disable_progress_bar (Possible options: --disable-noqa, --disable-progress-bar)\r\n```\n\n### How to reproduce\n\nSql file:\r\n```\r\nSELECT foo FROM bar;\r\n```\r\n\r\nCommand:\r\n```\r\nsqlfluff fix ${target} --dialect ansi --disable_progress_bar --force\r\n```\n\n### Dialect\n\nansi\n\n### Version\n\npython 3.10.3\r\nsqlfluff 1.4.0 and up appears to have this problem (tested through 1.4.2)\n\n### Configuration\n\nNo special configuration. Ran hermetically with `trunk`.\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [ ] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n--disable_progress_bar Flag Broken for Fix\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nI ran `sqlfluff fix ${target} --dialect ansi --disable_progress_bar --force` on version 1.4.0 and got an error with exit code 2. Running with `--disable-progress-bar` appears to work fine, but it appears that compatibility with underscores was broken in version 1.4.0.\n\n### Expected Behaviour\n\nShould run as expected, with no error and no progress bar.\n\n### Observed Behaviour\n\nExit code 2 and stderr:\r\n```\r\nUsage: sqlfluff fix [OPTIONS] [PATHS]...\r\n Try 'sqlfluff fix -h' for help.\r\n\r\n Error: No such option: --disable_progress_bar (Possible options: --disable-noqa, --disable-progress-bar)\r\n```\n\n### How to reproduce\n\nSql file:\r\n```\r\nSELECT foo FROM bar;\r\n```\r\n\r\nCommand:\r\n```\r\nsqlfluff fix ${target} --dialect ansi --disable_progress_bar --force\r\n```\n\n### Dialect\n\nansi\n\n### Version\n\npython 3.10.3\r\nsqlfluff 1.4.0 and up appears to have this problem (tested through 1.4.2)\n\n### Configuration\n\nNo special configuration. Ran hermetically with `trunk`.\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [ ] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "Looks like #3904 made `lint` work with both but updated `fix` to only accept `--disable-progress-bar`. I assume that was by accident. Should be relatively straightforward to fix by updating to match `lint`. \nLooks like #3904 made `lint` work with both but updated `fix` to only accept `--disable-progress-bar`. I assume that was by accident. Should be relatively straightforward to fix by updating to match `lint`. ", "created_at": "2022-12-11T16:33:31Z", "version": "1.3", "FAIL_TO_PASS": ["test/cli/commands_test.py::TestProgressBars::test_cli_fix_disabled_progress_bar_deprecated_option"], "PASS_TO_PASS": ["test/cli/commands_test.py::test__cli__command_directed", "test/cli/commands_test.py::test__cli__command_dialect", "test/cli/commands_test.py::test__cli__command_no_dialect", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_explicit_warning", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_implicit_warning", "test/cli/commands_test.py::test__cli__command_dialect_legacy", "test/cli/commands_test.py::test__cli__command_extra_config_fail", "test/cli/commands_test.py::test__cli__command_lint_stdin[command0]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command1]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command2]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command3]", "test/cli/commands_test.py::test__cli__command_render_stdin", "test/cli/commands_test.py::test__cli__command_lint_parse[command0]", "test/cli/commands_test.py::test__cli__command_lint_parse[command1]", "test/cli/commands_test.py::test__cli__command_lint_parse[command2]", "test/cli/commands_test.py::test__cli__command_lint_parse[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command4]", "test/cli/commands_test.py::test__cli__command_lint_parse[command5]", "test/cli/commands_test.py::test__cli__command_lint_parse[command6]", "test/cli/commands_test.py::test__cli__command_lint_parse[command7]", "test/cli/commands_test.py::test__cli__command_lint_parse[command8]", "test/cli/commands_test.py::test__cli__command_lint_parse[command9]", "test/cli/commands_test.py::test__cli__command_lint_parse[command10]", "test/cli/commands_test.py::test__cli__command_lint_parse[command11]", "test/cli/commands_test.py::test__cli__command_lint_parse[command12]", "test/cli/commands_test.py::test__cli__command_lint_parse[command13]", "test/cli/commands_test.py::test__cli__command_lint_parse[command14]", "test/cli/commands_test.py::test__cli__command_lint_parse[command15]", "test/cli/commands_test.py::test__cli__command_lint_parse[command16]", "test/cli/commands_test.py::test__cli__command_lint_parse[command17]", "test/cli/commands_test.py::test__cli__command_lint_parse[command18]", "test/cli/commands_test.py::test__cli__command_lint_parse[command19]", "test/cli/commands_test.py::test__cli__command_lint_parse[command20]", "test/cli/commands_test.py::test__cli__command_lint_parse[command21]", "test/cli/commands_test.py::test__cli__command_lint_parse[command22]", "test/cli/commands_test.py::test__cli__command_lint_parse[command23]", "test/cli/commands_test.py::test__cli__command_lint_parse[command24]", "test/cli/commands_test.py::test__cli__command_lint_parse[command25]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command0-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command1-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command2-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command3-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command4-1]", "test/cli/commands_test.py::test__cli__command_lint_warning_explicit_file_ignored", "test/cli/commands_test.py::test__cli__command_lint_skip_ignore_files", "test/cli/commands_test.py::test__cli__command_lint_ignore_local_config", "test/cli/commands_test.py::test__cli__command_lint_warning", "test/cli/commands_test.py::test__cli__command_versioning", "test/cli/commands_test.py::test__cli__command_version", "test/cli/commands_test.py::test__cli__command_rules", "test/cli/commands_test.py::test__cli__command_dialects", "test/cli/commands_test.py::test__cli__command__fix[L001-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/whitespace_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L003-test/fixtures/linter/indentation_error_hard.sql]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_templating_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error_FIX_EVEN_UNPARSABLE]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[2_files_with_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-True]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-True]", "test/cli/commands_test.py::test__cli__fix_loop_limit_behavior[--", "test/cli/commands_test.py::test__cli__command_fix_stdin[select", "test/cli/commands_test.py::test__cli__command_fix_stdin[", "test/cli/commands_test.py::test__cli__command_fix_stdin[SELECT", "test/cli/commands_test.py::test__cli__command_fix_stdin_logging_to_stderr", "test/cli/commands_test.py::test__cli__command_fix_stdin_safety", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[create", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[select", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-y-0-0]", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-n-1-1]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-json]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[select", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[SElect", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command0]", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command1]", "test/cli/commands_test.py::test__cli__command_lint_nocolor", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation_native", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation-native]", "test/cli/commands_test.py::test___main___help", "test/cli/commands_test.py::test_encoding[utf-8-ascii]", "test/cli/commands_test.py::test_encoding[utf-8-sig-UTF-8-SIG]", "test/cli/commands_test.py::test_encoding[utf-32-UTF-32]", "test/cli/commands_test.py::test_cli_encoding[utf-8-command-line-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-command-line-True]", "test/cli/commands_test.py::test_cli_encoding[utf-8-config-file-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-config-file-True]", "test/cli/commands_test.py::test_cli_no_disable_noqa_flag", "test/cli/commands_test.py::test_cli_disable_noqa_flag", "test/cli/commands_test.py::test_cli_get_default_config", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar_deprecated_option", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_paths", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_files", "test/cli/commands_test.py::TestProgressBars::test_cli_fix_disabled_progress_bar", "test/cli/commands_test.py::test__cli__fix_multiple_errors_no_show_errors", "test/cli/commands_test.py::test__cli__fix_multiple_errors_show_errors", "test/cli/commands_test.py::test__cli__multiple_files__fix_multiple_errors_show_errors", "test/cli/commands_test.py::test__cli__render_fail", "test/cli/commands_test.py::test__cli__render_pass"], "environment_setup_commit": "dc59c2a5672aacedaf91f0e6129b467eefad331b"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-3354", "base_commit": "36e89cbf2d13d5d95d2430f905a2fd122cf103c7", "patch": "diff --git a/src/sqlfluff/core/templaters/placeholder.py b/src/sqlfluff/core/templaters/placeholder.py\n--- a/src/sqlfluff/core/templaters/placeholder.py\n+++ b/src/sqlfluff/core/templaters/placeholder.py\n@@ -21,6 +21,8 @@\n KNOWN_STYLES = {\n # e.g. WHERE bla = :name\n \"colon\": regex.compile(r\"(?\\w+)(?!:)\", regex.UNICODE),\n+ # e.g. WHERE bla = table:name - use with caution as more prone to false positives\n+ \"colon_nospaces\": regex.compile(r\":(?P\\w+)\", regex.UNICODE),\n # e.g. WHERE bla = :2\n \"numeric_colon\": regex.compile(\n r\"(?\\d+)\", regex.UNICODE\n@@ -29,8 +31,10 @@\n \"pyformat\": regex.compile(\n r\"(?[\\w_]+)\\)s\", regex.UNICODE\n ),\n- # e.g. WHERE bla = $name\n- \"dollar\": regex.compile(r\"(?[\\w_]+)\", regex.UNICODE),\n+ # e.g. WHERE bla = $name or WHERE bla = ${name}\n+ \"dollar\": regex.compile(\n+ r\"(?[\\w_]+)}?\", regex.UNICODE\n+ ),\n # e.g. WHERE bla = ?\n \"question_mark\": regex.compile(r\"(? %(date)s\n+ AND someflag = %(someflag)s\n+ LIMIT %(limit)s\n \"\"\",\n \"pyformat\",\n \"\"\"\n@@ -130,10 +146,11 @@ def test__templater_raw():\n FROM users_data\n WHERE (city_id) IN (1, 2, 3, 45)\n AND date > '2020-10-01'\n+ AND someflag = False\n+ LIMIT 15\n \"\"\",\n dict(\n- city_id=\"(1, 2, 3, 45)\",\n- date=\"'2020-10-01'\",\n+ city_id=\"(1, 2, 3, 45)\", date=\"'2020-10-01'\", limit=15, someflag=False\n ),\n ),\n (\n@@ -142,6 +159,7 @@ def test__templater_raw():\n FROM users_data\n WHERE (city_id) IN $city_id\n AND date > $date\n+ OR date = ${date}\n \"\"\",\n \"dollar\",\n \"\"\"\n@@ -149,6 +167,7 @@ def test__templater_raw():\n FROM users_data\n WHERE (city_id) IN (1, 2, 3, 45)\n AND date > '2020-10-01'\n+ OR date = '2020-10-01'\n \"\"\",\n dict(\n city_id=\"(1, 2, 3, 45)\",\n@@ -221,6 +240,7 @@ def test__templater_raw():\n \"colon_simple_substitution\",\n \"colon_accept_block_at_end\",\n \"colon_tuple_substitution\",\n+ \"colon_nospaces\",\n \"question_mark\",\n \"numeric_colon\",\n \"pyformat\",\n", "problem_statement": "TypeError when using integer placeholder\n### Search before asking\r\n\r\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\r\n\r\n\r\n### What Happened\r\n\r\nAn exception occurs when trying to use integer substituents.\r\n\r\n### Expected Behaviour\r\n\r\nWork without errors.\r\n\r\n### Observed Behaviour\r\n\r\n\r\nAn exception occurs:\r\n```\r\n ...\r\n File \"venv/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py\", line 816, in render_file\r\n return self.render_string(raw_file, fname, config, encoding)\r\n File \"venv/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py\", line 787, in render_string\r\n templated_file, templater_violations = self.templater.process(\r\n File \"venv/lib/python3.9/site-packages/sqlfluff/core/templaters/placeholder.py\", line 183, in process\r\n start_template_pos, start_template_pos + len(replacement), None\r\nTypeError: object of type 'int' has no len()\r\n\r\n```\r\n\r\n### How to reproduce\r\n\r\n1. Create a file `example.sql`:\r\n```\r\nSELECT 1\r\nLIMIT %(capacity)s;\r\n```\r\n2. Copy `.sqlfluff` from the Configuration section\r\n3. Run `sqlfluff lint --dialect postgres example.sql`\r\n\r\n### Dialect\r\n\r\npostgres\r\n\r\n### Version\r\n\r\nsqlfluff, version 0.13.1\r\n\r\n### Configuration\r\n\r\n```\r\n[sqlfluff]\r\nexclude_rules = L031\r\ntemplater = placeholder\r\n\r\n[sqlfluff:templater:placeholder]\r\nparam_style = pyformat\r\ncapacity = 15\r\n```\r\n\r\n### Are you willing to work on and submit a PR to address the issue?\r\n\r\n- [ ] Yes I am willing to submit a PR!\r\n\r\n### Code of Conduct\r\n\r\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\r\n\nSupport Postgres-style variable substitution\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### Description\n\nThe Postgres `psql` utility supports flavor of colon-style variable substitution that currently confuses sqlfluff. E.g.,\r\n\r\n```sql\r\nALTER TABLE name:variable RENAME TO name;\r\n```\r\n\r\nRunning the above through sqlfluff produces this output:\r\n\r\n```\r\nsqlfluff lint --dialect postgres 2.sql\r\n== [2.sql] FAIL\r\nL: 1 | P: 1 | PRS | Line 1, Position 1: Found unparsable section: 'ALTER\r\n | TABLE name:variable RENAME TO name...'\r\n```\n\n### Use case\n\nI would like it if in the above the string \"name:variable\" were considered a valid table name (and other identifiers similarly).\n\n### Dialect\n\nThis applies to the Postgres dialect.\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [X] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "\nThis sounds more like a templater feature than a dialect feature. Does psql allow variables to contain SQL fragments, e.g.: `WHERE foo = '3'`?\n> This sounds more like a templater feature than a dialect feature.\r\n\r\nTrue! After looking over the code some, that may well be the right place to implement this.\r\n\r\n> Does psql allow variables to contain SQL fragments, e.g.: WHERE foo = '3'?\r\n\r\nYes. E.g.,\r\n\r\n```\r\n% psql -v expression='2 + 2'\r\npsql (14.2, server 10.18)\r\nSSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, bits: 256, compression: off)\r\nType \"help\" for help.\r\n\r\ndb=> select :expression;\r\n ?column?\r\n----------\r\n 4\r\n(1 row)\r\n\r\ndb=> select 5:expression;\r\n ?column?\r\n----------\r\n 54\r\n(1 row)\r\n```\r\n\r\nMore at the [docs](https://www.postgresql.org/docs/current/app-psql.html#APP-PSQL-VARIABLES).", "created_at": "2022-05-17T11:50:34Z", "version": "0.12", "FAIL_TO_PASS": ["test/core/templaters/placeholder_test.py::test__templater_param_style[colon_nospaces]", "test/core/templaters/placeholder_test.py::test__templater_param_style[pyformat]", "test/core/templaters/placeholder_test.py::test__templater_param_style[dollar]"], "PASS_TO_PASS": ["test/core/templaters/placeholder_test.py::test__templater_raw", "test/core/templaters/placeholder_test.py::test__templater_param_style[no_changes]", "test/core/templaters/placeholder_test.py::test__templater_param_style[colon_simple_substitution]", "test/core/templaters/placeholder_test.py::test__templater_param_style[colon_accept_block_at_end]", "test/core/templaters/placeholder_test.py::test__templater_param_style[colon_tuple_substitution]", "test/core/templaters/placeholder_test.py::test__templater_param_style[question_mark]", "test/core/templaters/placeholder_test.py::test__templater_param_style[numeric_colon]", "test/core/templaters/placeholder_test.py::test__templater_param_style[numeric_dollar]", "test/core/templaters/placeholder_test.py::test__templater_param_style[percent]", "test/core/templaters/placeholder_test.py::test__templater_param_style[ampersand]", "test/core/templaters/placeholder_test.py::test__templater_custom_regex", "test/core/templaters/placeholder_test.py::test__templater_exception", "test/core/templaters/placeholder_test.py::test__templater_setup", "test/core/templaters/placeholder_test.py::test__templater_styles"], "environment_setup_commit": "8f6fd1d8a8d69b2c463fbcf5bd1131c47f12ad88"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-3700", "base_commit": "1000cf1beae75186cadf3a586c87e86e9f30ecb2", "patch": "diff --git a/src/sqlfluff/core/parser/segments/base.py b/src/sqlfluff/core/parser/segments/base.py\n--- a/src/sqlfluff/core/parser/segments/base.py\n+++ b/src/sqlfluff/core/parser/segments/base.py\n@@ -1749,6 +1749,11 @@ def add(self, value): # MutableSet\n \"\"\"Add an element.\"\"\"\n self.map[self.key(value)] = value\n \n+ def update(self, value):\n+ \"\"\"Add elements in 'value'.\"\"\"\n+ for v in value:\n+ self.add(v)\n+\n def discard(self, value): # MutableSet\n \"\"\"Remove an element. Do not raise an exception if absent.\"\"\"\n self.map.pop(self.key(value), None) # pragma: no cover\ndiff --git a/src/sqlfluff/rules/L028.py b/src/sqlfluff/rules/L028.py\n--- a/src/sqlfluff/rules/L028.py\n+++ b/src/sqlfluff/rules/L028.py\n@@ -3,8 +3,9 @@\n from typing import Iterator, List, Optional, Set\n \n from sqlfluff.core.dialects.common import AliasInfo, ColumnAliasInfo\n-from sqlfluff.core.parser.segments.base import BaseSegment\n+from sqlfluff.core.parser.segments.base import BaseSegment, IdentitySet\n from sqlfluff.core.parser.segments.raw import SymbolSegment\n+from sqlfluff.utils.analysis.select import SelectStatementColumnsAndTables\n from sqlfluff.utils.analysis.select_crawler import Query, SelectCrawler\n from sqlfluff.core.rules import (\n BaseRule,\n@@ -99,12 +100,16 @@ def _eval(self, context: RuleContext) -> EvalResultType:\n \n if not FunctionalContext(context).parent_stack.any(sp.is_type(*_START_TYPES)):\n crawler = SelectCrawler(context.segment, context.dialect)\n+ visited: IdentitySet = IdentitySet()\n if crawler.query_tree:\n # Recursively visit and check each query in the tree.\n- return list(self._visit_queries(crawler.query_tree))\n+ return list(self._visit_queries(crawler.query_tree, visited))\n return None\n \n- def _visit_queries(self, query: Query) -> Iterator[LintResult]:\n+ def _visit_queries(\n+ self, query: Query, visited: IdentitySet\n+ ) -> Iterator[LintResult]:\n+ select_info: Optional[SelectStatementColumnsAndTables] = None\n if query.selectables:\n select_info = query.selectables[0].select_info\n # How many table names are visible from here? If more than one then do\n@@ -138,8 +143,24 @@ def _visit_queries(self, query: Query) -> Iterator[LintResult]:\n self._fix_inconsistent_to,\n fixable,\n )\n- for child in query.children:\n- yield from self._visit_queries(child)\n+ children = list(query.children)\n+ # 'query.children' includes CTEs and \"main\" queries, but not queries in\n+ # the \"FROM\" list. We want to visit those as well.\n+ if select_info:\n+ for a in select_info.table_aliases:\n+ for q in SelectCrawler.get(query, a.from_expression_element):\n+ if not isinstance(q, Query):\n+ continue\n+ # Check for previously visited selectables to avoid possible\n+ # infinite recursion, e.g.:\n+ # WITH test1 AS (SELECT i + 1, j + 1 FROM test1)\n+ # SELECT * FROM test1;\n+ if any(s.selectable in visited for s in q.selectables):\n+ continue\n+ visited.update(s.selectable for s in q.selectables)\n+ children.append(q)\n+ for child in children:\n+ yield from self._visit_queries(child, visited)\n \n \n def _check_references(\ndiff --git a/src/sqlfluff/rules/L042.py b/src/sqlfluff/rules/L042.py\n--- a/src/sqlfluff/rules/L042.py\n+++ b/src/sqlfluff/rules/L042.py\n@@ -2,7 +2,7 @@\n import copy\n from functools import partial\n from typing import (\n- Generator,\n+ Iterator,\n List,\n NamedTuple,\n Optional,\n@@ -12,7 +12,9 @@\n TypeVar,\n cast,\n )\n+\n from sqlfluff.core.dialects.base import Dialect\n+from sqlfluff.core.dialects.common import AliasInfo\n from sqlfluff.core.parser.segments.base import BaseSegment\n from sqlfluff.core.parser.segments.raw import (\n CodeSegment,\n@@ -21,8 +23,15 @@\n SymbolSegment,\n WhitespaceSegment,\n )\n-from sqlfluff.core.rules import BaseRule, LintFix, LintResult, RuleContext\n+from sqlfluff.core.rules import (\n+ BaseRule,\n+ EvalResultType,\n+ LintFix,\n+ LintResult,\n+ RuleContext,\n+)\n from sqlfluff.utils.analysis.select import get_select_statement_info\n+from sqlfluff.utils.analysis.select_crawler import Query, Selectable, SelectCrawler\n from sqlfluff.core.rules.crawlers import SegmentSeekerCrawler\n from sqlfluff.core.rules.doc_decorators import (\n document_configuration,\n@@ -51,10 +60,11 @@\n \n \n class _NestedSubQuerySummary(NamedTuple):\n- parent_clause_type: str\n- parent_select_segments: Segments\n- clause_segments: Segments\n- subquery: BaseSegment\n+ query: Query\n+ selectable: Selectable\n+ table_alias: AliasInfo\n+ sc: SelectCrawler\n+ select_source_names: Set[str]\n \n \n @document_groups\n@@ -107,137 +117,164 @@ class Rule_L042(BaseRule):\n \"both\": [\"join_clause\", \"from_expression_element\"],\n }\n \n- def _eval(self, context: RuleContext) -> Optional[List[LintResult]]:\n+ def _eval(self, context: RuleContext) -> EvalResultType:\n \"\"\"Join/From clauses should not contain subqueries. Use CTEs instead.\"\"\"\n self.forbid_subquery_in: str\n- parent_types = self._config_mapping[self.forbid_subquery_in]\n- segment = FunctionalContext(context).segment\n- parent_stack = FunctionalContext(context).parent_stack\n+ functional_context = FunctionalContext(context)\n+ segment = functional_context.segment\n+ parent_stack = functional_context.parent_stack\n+ is_select = segment.all(is_type(*_SELECT_TYPES))\n is_select_child = parent_stack.any(is_type(*_SELECT_TYPES))\n- if is_select_child:\n+ if not is_select or is_select_child:\n # Nothing to do.\n return None\n \n- # Gather all possible offending Elements in one crawl\n- nested_subqueries: List[_NestedSubQuerySummary] = []\n- selects = segment.recursive_crawl(*_SELECT_TYPES, recurse_into=True)\n- for select in selects.iterate_segments():\n- for res in _find_nested_subqueries(select, context.dialect):\n- if res.parent_clause_type not in parent_types:\n- continue\n- nested_subqueries.append(res)\n+ crawler = SelectCrawler(context.segment, context.dialect)\n+ assert crawler.query_tree\n+\n+ # generate an instance which will track and shape our output CTE\n+ ctes = _CTEBuilder()\n+ # Init the output/final select &\n+ # populate existing CTEs\n+ for cte in crawler.query_tree.ctes.values():\n+ ctes.insert_cte(cte.cte_definition_segment) # type: ignore\n+\n+ is_with = segment.all(is_type(\"with_compound_statement\"))\n+ # TODO: consider if we can fix recursive CTEs\n+ is_recursive = is_with and len(segment.children(is_keyword(\"recursive\"))) > 0\n+ case_preference = _get_case_preference(segment)\n+ output_select = segment\n+ if is_with:\n+ output_select = segment.children(\n+ is_type(\n+ \"set_expression\",\n+ \"select_statement\",\n+ )\n+ )\n \n- if not nested_subqueries:\n- return None\n # If there are offending elements calculate fixes\n- return _calculate_fixes(\n+ clone_map = SegmentCloneMap(segment[0])\n+ result = self._lint_query(\n dialect=context.dialect,\n- root_select=segment,\n- nested_subqueries=nested_subqueries,\n- parent_stack=parent_stack,\n+ query=crawler.query_tree,\n+ ctes=ctes,\n+ case_preference=case_preference,\n+ clone_map=clone_map,\n )\n \n-\n-def _calculate_fixes(\n- dialect: Dialect,\n- root_select: Segments,\n- nested_subqueries: List[_NestedSubQuerySummary],\n- parent_stack: Segments,\n-) -> List[LintResult]:\n- \"\"\"Given the Root select and the offending subqueries calculate fixes.\"\"\"\n- is_with = root_select.all(is_type(\"with_compound_statement\"))\n- # TODO: consider if we can fix recursive CTEs\n- is_recursive = is_with and len(root_select.children(is_keyword(\"recursive\"))) > 0\n- case_preference = _get_case_preference(root_select)\n- # generate an instance which will track and shape our output CTE\n- ctes = _CTEBuilder()\n- # Init the output/final select &\n- # populate existing CTEs\n- for cte in root_select.children(is_type(\"common_table_expression\")):\n- assert isinstance(cte, CTEDefinitionSegment), \"TypeGuard\"\n- ctes.insert_cte(cte)\n-\n- output_select = root_select\n- if is_with:\n- output_select = root_select.children(\n- is_type(\n- \"set_expression\",\n- \"select_statement\",\n+ if result:\n+ lint_result, from_expression, alias_name, subquery_parent = result\n+ assert any(\n+ from_expression is seg for seg in subquery_parent.recursive_crawl_all()\n )\n- )\n+ this_seg_clone = clone_map[from_expression]\n+ new_table_ref = _create_table_ref(alias_name, context.dialect)\n+ this_seg_clone.segments = [new_table_ref]\n+ ctes.replace_with_clone(subquery_parent, clone_map)\n+\n+ # Issue 3617: In T-SQL (and possibly other dialects) the automated fix\n+ # leaves parentheses in a location that causes a syntax error. This is an\n+ # unusual corner case. For simplicity, we still generate the lint warning\n+ # but don't try to generate a fix. Someone could look at this later (a\n+ # correct fix would involve removing the parentheses.)\n+ bracketed_ctas = [seg.type for seg in parent_stack[-2:]] == [\n+ \"create_table_statement\",\n+ \"bracketed\",\n+ ]\n+ if bracketed_ctas or ctes.has_duplicate_aliases() or is_recursive:\n+ # If we have duplicate CTE names just don't fix anything\n+ # Return the lint warnings anyway\n+ return lint_result\n+\n+ # Compute fix.\n+ edit = [\n+ ctes.compose_select(\n+ clone_map[output_select[0]],\n+ case_preference=case_preference,\n+ ),\n+ ]\n+ lint_result.fixes = [\n+ LintFix.replace(\n+ segment[0],\n+ edit_segments=edit,\n+ )\n+ ]\n+ return lint_result\n+ return None\n \n- lint_results: List[LintResult] = []\n- clone_map = SegmentCloneMap(root_select[0])\n- is_new_name = False\n- new_table_ref = None\n- for parent_type, _, this_seg, subquery in nested_subqueries:\n- alias_name, is_new_name = ctes.create_cte_alias(\n- this_seg.children(is_type(\"alias_expression\"))\n- )\n- new_cte = _create_cte_seg(\n- alias_name=alias_name,\n- subquery=clone_map[subquery],\n- case_preference=case_preference,\n- dialect=dialect,\n- )\n- ctes.insert_cte(new_cte)\n- this_seg_clone = clone_map[this_seg[0]]\n- assert this_seg_clone.pos_marker, \"TypeGuard\"\n- new_table_ref = _create_table_ref(alias_name, dialect)\n- this_seg_clone.segments = (new_table_ref,)\n- anchor = subquery\n- # Grab the first keyword or symbol in the subquery to use as the\n- # anchor. This makes the lint warning less likely to be filtered out\n- # if a bit of the subquery happens to be templated.\n- for seg in subquery.recursive_crawl(\"keyword\", \"symbol\"):\n- anchor = seg\n- break\n- res = LintResult(\n- anchor=anchor,\n- description=f\"{parent_type} clauses should not contain \"\n- \"subqueries. Use CTEs instead\",\n- fixes=[],\n- )\n- lint_results.append(res)\n-\n- # Issue 3617: In T-SQL (and possibly other dialects) the automated fix\n- # leaves parentheses in a location that causes a syntax error. This is an\n- # unusual corner case. For simplicity, we still generate the lint warning\n- # but don't try to generate a fix. Someone could look at this later (a\n- # correct fix would involve removing the parentheses.)\n- bracketed_ctas = [seg.type for seg in parent_stack[-2:]] == [\n- \"create_table_statement\",\n- \"bracketed\",\n- ]\n- if bracketed_ctas or ctes.has_duplicate_aliases() or is_recursive:\n- # If we have duplicate CTE names just don't fix anything\n- # Return the lint warnings anyway\n- return lint_results\n-\n- # Add fixes to the last result only\n- edit = [\n- ctes.compose_select(\n- clone_map[output_select[0]],\n- case_preference=case_preference,\n- ),\n- ]\n- lint_results[-1].fixes = [\n- LintFix.replace(\n- root_select[0],\n- edit_segments=edit,\n- )\n- ]\n- if is_new_name:\n- assert lint_results[0].fixes[0].edit\n- assert new_table_ref\n- # If we're creating a new CTE name but the CTE name does not appear in\n- # the fix, discard the lint error. This prevents the rule from looping,\n- # i.e. making the same fix repeatedly.\n- if not any(\n- seg.uuid == new_table_ref.uuid for seg in edit[0].recursive_crawl_all()\n- ):\n- lint_results[-1].fixes = []\n- return lint_results\n+ def _nested_subqueries(\n+ self, query: Query, dialect: Dialect\n+ ) -> Iterator[_NestedSubQuerySummary]:\n+ parent_types = self._config_mapping[self.forbid_subquery_in]\n+ for q in [query] + list(query.ctes.values()):\n+ for selectable in q.selectables:\n+ if not selectable.select_info:\n+ continue # pragma: no cover\n+ select_source_names = set()\n+ for a in selectable.select_info.table_aliases:\n+ # For each table in FROM, return table name and any alias.\n+ if a.ref_str:\n+ select_source_names.add(a.ref_str)\n+ if a.object_reference:\n+ select_source_names.add(a.object_reference.raw)\n+ for table_alias in selectable.select_info.table_aliases:\n+ sc = SelectCrawler(table_alias.from_expression_element, dialect)\n+ if sc.query_tree:\n+ path_to = selectable.selectable.path_to(\n+ table_alias.from_expression_element\n+ )\n+ if not any(seg.is_type(*parent_types) for seg in path_to):\n+ continue\n+ if _is_correlated_subquery(\n+ Segments(sc.query_tree.selectables[0].selectable),\n+ select_source_names,\n+ dialect,\n+ ):\n+ continue\n+ yield _NestedSubQuerySummary(\n+ q, selectable, table_alias, sc, select_source_names\n+ )\n+\n+ def _lint_query(\n+ self,\n+ dialect: Dialect,\n+ query: Query,\n+ ctes: \"_CTEBuilder\",\n+ case_preference,\n+ clone_map,\n+ ) -> Optional[Tuple[LintResult, BaseSegment, str, BaseSegment]]:\n+ \"\"\"Given the root query, compute lint warnings.\"\"\"\n+ nsq: _NestedSubQuerySummary\n+ for nsq in self._nested_subqueries(query, dialect):\n+ alias_name, is_new_name = ctes.create_cte_alias(nsq.table_alias)\n+ anchor = nsq.table_alias.from_expression_element.segments[0]\n+ new_cte = _create_cte_seg(\n+ alias_name=alias_name,\n+ subquery=clone_map[anchor],\n+ case_preference=case_preference,\n+ dialect=dialect,\n+ )\n+ ctes.insert_cte(new_cte)\n+\n+ # Grab the first keyword or symbol in the subquery to\n+ # use as the anchor. This makes the lint warning less\n+ # likely to be filtered out if a bit of the subquery\n+ # happens to be templated.\n+ anchor = next(anchor.recursive_crawl(\"keyword\", \"symbol\"))\n+ res = LintResult(\n+ anchor=anchor,\n+ description=f\"{nsq.query.selectables[0].selectable.type} clauses \"\n+ \"should not contain subqueries. Use CTEs instead\",\n+ fixes=[],\n+ )\n+ if len(nsq.query.selectables) == 1:\n+ return (\n+ res,\n+ nsq.table_alias.from_expression_element,\n+ alias_name,\n+ nsq.query.selectables[0].selectable,\n+ )\n+ return None\n \n \n def _get_first_select_statement_descendant(\n@@ -252,27 +289,6 @@ def _get_first_select_statement_descendant(\n return None # pragma: no cover\n \n \n-def _get_sources_from_select(segment: BaseSegment, dialect: Dialect) -> Set[str]:\n- \"\"\"Given segment, return set of table or alias names it queries from.\"\"\"\n- result = set()\n- select = None\n- if segment.is_type(\"select_statement\"):\n- select = segment\n- elif segment.is_type(\"with_compound_statement\"):\n- # For WITH statement, process the main query underneath.\n- select = _get_first_select_statement_descendant(segment)\n- if select and select.is_type(\"select_statement\"):\n- select_info = get_select_statement_info(select, dialect)\n- if select_info:\n- for a in select_info.table_aliases:\n- # For each table in FROM, return table name and any alias.\n- if a.ref_str:\n- result.add(a.ref_str)\n- if a.object_reference:\n- result.add(a.object_reference.raw)\n- return result\n-\n-\n def _is_correlated_subquery(\n nested_select: Segments, select_source_names: Set[str], dialect: Dialect\n ):\n@@ -280,8 +296,6 @@ def _is_correlated_subquery(\n \n https://en.wikipedia.org/wiki/Correlated_subquery\n \"\"\"\n- if not nested_select:\n- return False # pragma: no cover\n select_statement = _get_first_select_statement_descendant(nested_select[0])\n if not select_statement:\n return False # pragma: no cover\n@@ -298,51 +312,6 @@ def _is_correlated_subquery(\n return False\n \n \n-def _find_nested_subqueries(\n- select: Segments,\n- dialect: Dialect,\n-) -> Generator[_NestedSubQuerySummary, None, None]:\n- \"\"\"Find possible offending elements and return enough to fix them.\"\"\"\n- select_types = [\n- \"with_compound_statement\",\n- \"set_expression\",\n- \"select_statement\",\n- ]\n- from_clause = select.children().first(is_type(\"from_clause\")).children()\n- offending_types = [\"join_clause\", \"from_expression_element\"]\n- select_source_names = _get_sources_from_select(select[0], dialect)\n-\n- # Match any of the types we care about\n- for this_seg in from_clause.children(is_type(*offending_types)).iterate_segments():\n- parent_type = this_seg[0].get_type()\n- # Ensure we are at the right depth (from_expression_element)\n- if not this_seg.all(is_type(\"from_expression_element\")):\n- this_seg = this_seg.children(\n- is_type(\"from_expression_element\"),\n- )\n-\n- table_expression_el = this_seg.children(\n- is_type(\"table_expression\"),\n- )\n-\n- # Is it bracketed? If so, lint that instead.\n- bracketed_expression = table_expression_el.children(\n- is_type(\"bracketed\"),\n- )\n- nested_select = bracketed_expression or table_expression_el\n- # If we find a child with a \"problem\" type, raise an issue.\n- # If not, we're fine.\n- seg = nested_select.children(is_type(*select_types))\n- if not seg:\n- # If there is no match there is no error\n- continue\n- # Type, parent_select, parent_sequence\n- if not _is_correlated_subquery(nested_select, select_source_names, dialect):\n- yield _NestedSubQuerySummary(\n- parent_type, select, this_seg, table_expression_el[0]\n- )\n-\n-\n class _CTEBuilder:\n \"\"\"Gather CTE parts, maintain order and track naming/aliasing.\"\"\"\n \n@@ -369,7 +338,9 @@ def has_duplicate_aliases(self) -> bool:\n def insert_cte(self, cte: CTEDefinitionSegment):\n \"\"\"Add a new CTE to the list as late as possible but before all its parents.\"\"\"\n # This should still have the position markers of its true position\n- inbound_subquery = Segments(cte).children().last()\n+ inbound_subquery = (\n+ Segments(cte).children().last(lambda seg: bool(seg.pos_marker))\n+ )\n insert_position = next(\n (\n i\n@@ -381,14 +352,11 @@ def insert_cte(self, cte: CTEDefinitionSegment):\n \n self.ctes.insert(insert_position, cte)\n \n- def create_cte_alias(\n- self, alias_segment: Optional[Segments] = None\n- ) -> Tuple[str, bool]:\n+ def create_cte_alias(self, alias: Optional[AliasInfo]) -> Tuple[str, bool]:\n \"\"\"Find or create the name for the next CTE.\"\"\"\n- if alias_segment:\n+ if alias and alias.aliased and alias.ref_str:\n # If we know the name use it\n- name = alias_segment.children().last()[0].raw\n- return name, False\n+ return alias.ref_str, False\n \n self.name_idx = self.name_idx + 1\n name = f\"prep_{self.name_idx}\"\n@@ -398,7 +366,7 @@ def create_cte_alias(\n return name, True\n \n def get_cte_segments(self) -> List[BaseSegment]:\n- \"\"\"Return a valid list of CTES with required padding Segements.\"\"\"\n+ \"\"\"Return a valid list of CTES with required padding segments.\"\"\"\n cte_segments: List[BaseSegment] = []\n for cte in self.ctes:\n cte_segments = cte_segments + [\n@@ -439,16 +407,24 @@ def compose_select(self, output_select: BaseSegment, case_preference: str):\n )\n return new_select\n \n+ def replace_with_clone(self, segment, clone_map):\n+ for idx, cte in enumerate(self.ctes):\n+ if any(segment is seg for seg in cte.recursive_crawl_all()):\n+ self.ctes[idx] = clone_map[self.ctes[idx]]\n+ return\n+\n \n def _is_child(maybe_parent: Segments, maybe_child: Segments) -> bool:\n \"\"\"Is the child actually between the start and end markers of the parent.\"\"\"\n- assert len(maybe_child) == 1, \"Cannot assess Childness of multiple Segments\"\n- assert len(maybe_parent) == 1, \"Cannot assess Childness of multiple Parents\"\n+ assert (\n+ len(maybe_child) == 1\n+ ), \"Cannot assess child relationship of multiple segments\"\n+ assert (\n+ len(maybe_parent) == 1\n+ ), \"Cannot assess child relationship of multiple parents\"\n child_markers = maybe_child[0].pos_marker\n parent_pos = maybe_parent[0].pos_marker\n- if not parent_pos or not child_markers:\n- return False # pragma: no cover\n-\n+ assert parent_pos and child_markers\n if child_markers < parent_pos.start_point_marker():\n return False # pragma: no cover\n \ndiff --git a/src/sqlfluff/utils/analysis/select_crawler.py b/src/sqlfluff/utils/analysis/select_crawler.py\n--- a/src/sqlfluff/utils/analysis/select_crawler.py\n+++ b/src/sqlfluff/utils/analysis/select_crawler.py\n@@ -33,8 +33,13 @@ class Selectable:\n \"\"\"A \"SELECT\" query segment.\"\"\"\n \n selectable: BaseSegment\n+ parent: Optional[BaseSegment]\n dialect: Dialect\n \n+ def as_str(self) -> str:\n+ \"\"\"String representation for logging/testing.\"\"\"\n+ return self.selectable.raw\n+\n @cached_property\n def select_info(self):\n \"\"\"Returns SelectStatementColumnsAndTables on the SELECT.\"\"\"\n@@ -112,7 +117,7 @@ def find_alias(self, table: str) -> Optional[AliasInfo]:\n \"\"\"Find corresponding table_aliases entry (if any) matching \"table\".\"\"\"\n alias_info = [\n t\n- for t in self.select_info.table_aliases\n+ for t in (self.select_info.table_aliases if self.select_info else [])\n if t.aliased and t.ref_str == table\n ]\n assert len(alias_info) <= 1\n@@ -131,8 +136,24 @@ class Query:\n parent: Optional[\"Query\"] = field(default=None)\n # Children (could be CTE, subselect, or other).\n children: List[\"Query\"] = field(default_factory=list)\n+ cte_definition_segment: Optional[BaseSegment] = field(default=None)\n cte_name_segment: Optional[BaseSegment] = field(default=None)\n \n+ def as_json(self) -> Dict:\n+ \"\"\"JSON representation for logging/testing.\"\"\"\n+ result = {}\n+ if self.query_type != QueryType.Simple:\n+ result[\"query_type\"] = self.query_type.name\n+ if self.selectables:\n+ result[\"selectables\"] = [\n+ s.as_str() for s in self.selectables\n+ ] # type: ignore\n+ if self.ctes:\n+ result[\"ctes\"] = {\n+ k: v.as_json() for k, v in self.ctes.items()\n+ } # type: ignore\n+ return result\n+\n def lookup_cte(self, name: str, pop: bool = True) -> Optional[\"Query\"]:\n \"\"\"Look up a CTE by name, in the current or any parent scope.\"\"\"\n cte = self.ctes.get(name.upper())\n@@ -146,7 +167,7 @@ def lookup_cte(self, name: str, pop: bool = True) -> Optional[\"Query\"]:\n return None\n \n def crawl_sources(\n- self, segment: BaseSegment, recurse_into=True, pop=False\n+ self, segment: BaseSegment, recurse_into=True, pop=False, lookup_cte=True\n ) -> Generator[Union[str, \"Query\"], None, None]:\n \"\"\"Find SELECTs, table refs, or value table function calls in segment.\n \n@@ -154,20 +175,26 @@ def crawl_sources(\n references or function call strings, yield those.\n \"\"\"\n found_nested_select = False\n- for seg in segment.recursive_crawl(\n+ types = [\n \"table_reference\",\n \"set_expression\",\n \"select_statement\",\n \"values_clause\",\n- recurse_into=recurse_into,\n+ ]\n+ for event, path in SelectCrawler.visit_segments(\n+ segment, recurse_into=recurse_into\n ):\n+ seg = path[-1]\n+ if event == \"end\" or not seg.is_type(*types):\n+ continue\n+\n if seg is segment:\n # If the starting segment itself matches the list of types we're\n # searching for, recursive_crawl() will return it. Skip that.\n continue\n \n if seg.is_type(\"table_reference\"):\n- if not seg.is_qualified():\n+ if not seg.is_qualified() and lookup_cte:\n cte = self.lookup_cte(seg.raw, pop=pop)\n if cte:\n # It's a CTE.\n@@ -179,7 +206,15 @@ def crawl_sources(\n \"set_expression\", \"select_statement\", \"values_clause\"\n )\n found_nested_select = True\n- crawler = SelectCrawler(seg, self.dialect, parent=self)\n+ seg_ = Segments(*path[1:]).first(\n+ sp.is_type(\n+ \"from_expression_element\",\n+ \"set_expression\",\n+ \"select_statement\",\n+ \"values_clause\",\n+ )\n+ )[0]\n+ crawler = SelectCrawler(seg_, self.dialect, parent=self)\n # We know this will pass because we specified parent=self above.\n assert crawler.query_tree\n yield crawler.query_tree\n@@ -234,9 +269,10 @@ def finish_segment():\n except ValueError:\n pass\n \n- # Stores the last CTE name we saw, so we can associate it with the\n- # corresponding Query.\n- cte_name_segment: Optional[BaseSegment] = None\n+ # Stacks for CTE definition & names we've seen but haven't consumed yet,\n+ # so we can associate with the corresponding Query.\n+ cte_definition_segment_stack: List[BaseSegment] = []\n+ cte_name_segment_stack: List[BaseSegment] = []\n \n # Visit segment and all its children\n for event, path in SelectCrawler.visit_segments(segment):\n@@ -263,9 +299,17 @@ def finish_segment():\n # added to this Query later.\n query = self.query_class(QueryType.Simple, dialect)\n append_query(query)\n- else:\n+ # Ignore segments under a from_expression_element.\n+ # Those will be nested queries, and we're only\n+ # interested in CTEs and \"main\" queries, i.e.\n+ # standalones or those following a block of CTEs.\n+ elif not any(\n+ seg.is_type(\"from_expression_element\") for seg in path[1:]\n+ ):\n # It's a select_statement or values_clause.\n- selectable = Selectable(path[-1], dialect)\n+ selectable = Selectable(\n+ path[-1], path[-2] if len(path) >= 2 else None, dialect\n+ )\n # Determine if this is part of a set_expression.\n if len(path) >= 2 and path[-2].is_type(\"set_expression\"):\n # It's part of a set_expression. Append to the\n@@ -280,27 +324,37 @@ def finish_segment():\n append_query(query)\n else:\n # We're processing a \"with\" statement.\n- if cte_name_segment:\n+ if cte_name_segment_stack:\n # If we have a CTE name, this is the Query for that\n # name.\n query = self.query_class(\n QueryType.Simple,\n dialect,\n- cte_name_segment=cte_name_segment,\n+ cte_definition_segment=cte_definition_segment_stack[-1],\n+ cte_name_segment=cte_name_segment_stack[-1],\n )\n if path[-1].is_type(\n \"select_statement\", \"values_clause\", \"update_statement\"\n ):\n # Add to the Query object we just created.\n- query.selectables.append(Selectable(path[-1], dialect))\n+ query.selectables.append(\n+ Selectable(\n+ path[-1],\n+ path[-2] if len(path) >= 2 else None,\n+ dialect,\n+ )\n+ )\n else:\n # Processing a set_expression. Nothing\n # additional to do here; we'll add selectables\n # to the Query later when we encounter those\n # child segments.\n pass\n- query_stack[-1].ctes[cte_name_segment.raw_upper] = query\n- cte_name_segment = None\n+ query_stack[-1].ctes[\n+ cte_name_segment_stack[-1].raw_upper\n+ ] = query\n+ cte_definition_segment_stack.pop()\n+ cte_name_segment_stack.pop()\n append_query(query)\n else:\n # There's no CTE name, so we're probably processing\n@@ -311,7 +365,8 @@ def finish_segment():\n # interested in CTEs and \"main\" queries, i.e.\n # standalones or those following a block of CTEs.\n if not any(\n- seg.is_type(\"from_expression_element\") for seg in path\n+ seg.is_type(\"from_expression_element\")\n+ for seg in path[1:]\n ):\n if path[-1].is_type(\n \"select_statement\", \"update_statement\"\n@@ -319,7 +374,11 @@ def finish_segment():\n # Processing a select_statement. Add it to the\n # Query object on top of the stack.\n query_stack[-1].selectables.append(\n- Selectable(path[-1], dialect)\n+ Selectable(\n+ path[-1],\n+ path[-2] if len(path) >= 2 else None,\n+ dialect,\n+ )\n )\n else:\n # Processing a set_expression. Nothing\n@@ -328,13 +387,19 @@ def finish_segment():\n elif path[-1].is_type(\"with_compound_statement\"):\n # Beginning a \"with\" statement, i.e. a block of CTEs.\n query = self.query_class(QueryType.WithCompound, dialect)\n- if cte_name_segment:\n- query_stack[-1].ctes[cte_name_segment.raw_upper] = query\n- cte_name_segment = None\n+ if cte_name_segment_stack:\n+ query_stack[-1].ctes[\n+ cte_name_segment_stack[-1].raw_upper\n+ ] = query\n+ query.cte_definition_segment = cte_definition_segment_stack[-1]\n+ cte_definition_segment_stack.pop()\n+ cte_name_segment_stack.pop()\n append_query(query)\n elif path[-1].is_type(\"common_table_expression\"):\n- # This is a \"<> AS\". Grab the name for later.\n- cte_name_segment = path[-1].segments[0]\n+ # This is a \"<> AS\". Save definition segment and\n+ # name for later.\n+ cte_definition_segment_stack.append(path[-1])\n+ cte_name_segment_stack.append(path[-1].segments[0])\n elif event == \"end\":\n finish_segment()\n \n@@ -355,13 +420,14 @@ def get(cls, query: Query, segment: BaseSegment) -> List[Union[str, \"Query\"]]:\n return list(query.crawl_sources(segment, True))\n \n @classmethod\n- def visit_segments(cls, seg, path=None):\n+ def visit_segments(cls, seg, path=None, recurse_into=True):\n \"\"\"Recursively visit all segments.\"\"\"\n if path is None:\n path = []\n path.append(seg)\n yield \"start\", path\n- for seg in seg.segments:\n- yield from cls.visit_segments(seg, path)\n+ if recurse_into:\n+ for seg in seg.segments:\n+ yield from cls.visit_segments(seg, path, recurse_into)\n yield \"end\", path\n path.pop()\n", "test_patch": "diff --git a/test/fixtures/rules/std_rule_cases/L042.yml b/test/fixtures/rules/std_rule_cases/L042.yml\n--- a/test/fixtures/rules/std_rule_cases/L042.yml\n+++ b/test/fixtures/rules/std_rule_cases/L042.yml\n@@ -99,7 +99,7 @@ double_nested_fail:\n L042:\n forbid_subquery_in: both\n \n-double_nested_unfixable_cte_clash:\n+double_nested_fail_2:\n fail_str: |\n select\n a.x, a.y, b.z\n@@ -109,6 +109,20 @@ double_nested_unfixable_cte_clash:\n select x, z from p_cte\n ) as b\n ) as b on (a.x = b.x)\n+ fix_str: |\n+ with b as (\n+ select x, z from (\n+ select x, z from p_cte\n+ ) as b\n+ )\n+ select\n+ a.x, a.y, b.z\n+ from a\n+ join b on (a.x = b.x)\n+ violations_after_fix:\n+ - description: select_statement clauses should not contain subqueries. Use CTEs instead\n+ line_no: 2\n+ line_pos: 20\n configs:\n rules:\n L042:\n@@ -127,6 +141,23 @@ unfixable_cte_clash:\n select 1\n ) as b\n ) as c on (a.x = b.x)\n+ fix_str: |\n+ with \"b\" as (\n+ select x, z from p_cte\n+ ),\n+ c as (\n+ select x, z from (\n+ select 1\n+ ) as b\n+ )\n+ select\n+ a.x, a.y, b.z\n+ from a\n+ join c on (a.x = b.x)\n+ violations_after_fix:\n+ - description: select_statement clauses should not contain subqueries. Use CTEs instead\n+ line_no: 5\n+ line_pos: 20\n configs:\n rules:\n L042:\n@@ -458,10 +489,16 @@ issue_3572_correlated_subquery_3:\n issue_3598_avoid_looping_1:\n fail_str: |\n WITH cte1 AS (\n- \tSELECT a\n- \tFROM (SELECT a)\n+ SELECT a\n+ FROM (SELECT a)\n+ )\n+ SELECT a FROM cte1\n+ fix_str: |\n+ WITH prep_1 AS (SELECT a),\n+ cte1 AS (\n+ SELECT a\n+ FROM prep_1\n )\n-\n SELECT a FROM cte1\n configs:\n rules:\n@@ -474,8 +511,37 @@ issue_3598_avoid_looping_2:\n SELECT *\n FROM (SELECT * FROM mongo.temp)\n )\n-\n SELECT * FROM cte1\n+ fix_str: |\n+ WITH prep_1 AS (SELECT * FROM mongo.temp),\n+ cte1 AS (\n+ SELECT *\n+ FROM prep_1\n+ )\n+ SELECT * FROM cte1\n+ configs:\n+ rules:\n+ L042:\n+ forbid_subquery_in: both\n+\n+test_fail_subquery_in_cte:\n+ fail_str: |\n+ with b as (\n+ select x, z from (\n+ select x, z from p_cte\n+ )\n+ )\n+ select b.z\n+ from b\n+ fix_str: |\n+ with prep_1 as (\n+ select x, z from p_cte\n+ ),\n+ b as (\n+ select x, z from prep_1\n+ )\n+ select b.z\n+ from b\n configs:\n rules:\n L042:\ndiff --git a/test/utils/analysis/test_select_crawler.py b/test/utils/analysis/test_select_crawler.py\nnew file mode 100644\n--- /dev/null\n+++ b/test/utils/analysis/test_select_crawler.py\n@@ -0,0 +1,197 @@\n+\"\"\"Test the select_crawler module.\"\"\"\n+import pytest\n+\n+from sqlfluff.core.linter.linter import Linter\n+from sqlfluff.utils.analysis import select_crawler\n+\n+\n+@pytest.mark.parametrize(\n+ \"sql, expected_json\",\n+ [\n+ (\n+ # Test trivial query.\n+ \"select 1\",\n+ {\"selectables\": [\"select 1\"]},\n+ ),\n+ (\n+ # Test set expression.\n+ \"select 1 union select 2\",\n+ {\"selectables\": [\"select 1\", \"select 2\"]},\n+ ),\n+ (\n+ # Test multiple CTEs.\n+ \"with cte1 as (select 1 as x), cte2 as (select 2 as y) \"\n+ \"select * from cte1 join cte2 using (x)\",\n+ {\n+ \"ctes\": {\n+ \"CTE1\": {\"selectables\": [\"select 1 as x\"]},\n+ \"CTE2\": {\"selectables\": [\"select 2 as y\"]},\n+ },\n+ \"query_type\": \"WithCompound\",\n+ \"selectables\": [\"select * from cte1 join cte2 using (x)\"],\n+ },\n+ ),\n+ (\n+ # Nested CTEs (from L044 test suite)\n+ \"\"\"\n+ with a as (\n+ with b as (select 1 from c)\n+ select * from b\n+ )\n+ select * from a\n+ \"\"\",\n+ {\n+ \"ctes\": {\n+ \"A\": {\n+ \"ctes\": {\"B\": {\"selectables\": [\"select 1 from c\"]}},\n+ \"query_type\": \"WithCompound\",\n+ \"selectables\": [\"select * from b\"],\n+ }\n+ },\n+ \"query_type\": \"WithCompound\",\n+ \"selectables\": [\"select * from a\"],\n+ },\n+ ),\n+ (\n+ # Nested CTEs (from L044 test suite)\n+ \"\"\"\n+ with b as (select 1 from c)\n+ select * from (\n+ with a as (select * from b)\n+ select * from a\n+ )\n+ \"\"\",\n+ {\n+ \"ctes\": {\"B\": {\"selectables\": [\"select 1 from c\"]}},\n+ \"query_type\": \"WithCompound\",\n+ \"selectables\": [\n+ \"select * from (\\n\"\n+ \" with a as (select * from b)\\n\"\n+ \" select * from a\\n\"\n+ \" )\"\n+ ],\n+ },\n+ ),\n+ (\n+ # Test that subquery in \"from\" not included.\n+ \"select a.x from (select z from b)\",\n+ {\"selectables\": [\"select a.x from (select z from b)\"]},\n+ ),\n+ (\n+ # Test that subquery in \"from\" / \"join\" not included.\n+ \"select a.x from a join (select z from b) as b on (a.x = b.x)\",\n+ {\n+ \"selectables\": [\n+ \"select a.x from a join (select z from b) as b on (a.x = b.x)\"\n+ ]\n+ },\n+ ),\n+ (\n+ # In CTE main query, test that subquery in \"from\" not included.\n+ \"with prep as (select 1) select a.x from (select z from b)\",\n+ {\n+ \"ctes\": {\"PREP\": {\"selectables\": [\"select 1\"]}},\n+ \"query_type\": \"WithCompound\",\n+ \"selectables\": [\"select a.x from (select z from b)\"],\n+ },\n+ ),\n+ (\n+ # In CTE main query, test that subquery in \"from\" / \"join\" not included.\n+ \"with prep as (select 1) \"\n+ \"select a.x from a join (select z from b) as b on (a.x = b.x)\",\n+ {\n+ \"ctes\": {\"PREP\": {\"selectables\": [\"select 1\"]}},\n+ \"query_type\": \"WithCompound\",\n+ \"selectables\": [\n+ \"select a.x from a join (select z from b) as b on (a.x = \" \"b.x)\"\n+ ],\n+ },\n+ ),\n+ (\n+ \"\"\"with prep_1 as (\n+ with d as (\n+ select x, z from b\n+ )\n+ select * from d\n+)\n+select\n+ a.x, a.y, b.z\n+from a\n+join prep_1 using (x)\n+\"\"\",\n+ {\n+ \"ctes\": {\n+ \"PREP_1\": {\n+ \"ctes\": {\n+ \"D\": {\"selectables\": [\"select x, z from b\"]},\n+ },\n+ \"query_type\": \"WithCompound\",\n+ \"selectables\": [\"select * from d\"],\n+ }\n+ },\n+ \"query_type\": \"WithCompound\",\n+ \"selectables\": [\n+ \"select\\n a.x, a.y, b.z\\nfrom a\\njoin prep_1 using (x)\"\n+ ],\n+ },\n+ ),\n+ ],\n+)\n+def test_select_crawler_constructor(sql, expected_json):\n+ \"\"\"Test SelectCrawler when created using constructor.\"\"\"\n+ linter = Linter(dialect=\"ansi\")\n+ parsed = linter.parse_string(sql)\n+ segments = list(\n+ parsed.tree.recursive_crawl(\n+ \"with_compound_statement\",\n+ \"set_expression\",\n+ \"select_statement\",\n+ )\n+ )\n+ segment = segments[0]\n+ crawler = select_crawler.SelectCrawler(segment, linter.dialect)\n+ assert all(\n+ cte.cte_definition_segment is not None\n+ for cte in crawler.query_tree.ctes.values()\n+ )\n+ json_query_tree = crawler.query_tree.as_json()\n+ assert expected_json == json_query_tree\n+\n+\n+def test_select_crawler_nested():\n+ \"\"\"Test invoking with an outer from_expression_segment.\"\"\"\n+ sql = \"\"\"\n+select\n+ a.x, a.y, b.z\n+from a\n+join (\n+ with d as (\n+ select x, z from b\n+ )\n+ select * from d\n+) using (x)\n+ \"\"\"\n+ linter = Linter(dialect=\"ansi\")\n+ parsed = linter.parse_string(sql)\n+ segments = list(\n+ parsed.tree.recursive_crawl(\n+ \"with_compound_statement\",\n+ \"set_expression\",\n+ \"select_statement\",\n+ )\n+ )\n+ segment = segments[0]\n+ crawler = select_crawler.SelectCrawler(segment, linter.dialect)\n+ sc = select_crawler.SelectCrawler(\n+ crawler.query_tree.selectables[0]\n+ .select_info.table_aliases[1]\n+ .from_expression_element,\n+ linter.dialect,\n+ )\n+ assert sc.query_tree.as_json() == {\n+ \"selectables\": [\n+ \"select * from d\",\n+ ],\n+ \"ctes\": {\"D\": {\"selectables\": [\"select x, z from b\"]}},\n+ \"query_type\": \"WithCompound\",\n+ }\n", "problem_statement": "L042 loop limit on fixes reached when CTE itself contains a subquery\n### Search before asking\r\n\r\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\r\n\r\n\r\n### What Happened\r\n\r\nWhile running `sqlfluff fix --dialect snowflake` on a sql file, I get \r\n```\r\n==== finding fixable violations ====\r\nWARNING Loop limit on fixes reached [10]. \r\n==== no fixable linting violations found ==== \r\nAll Finished \ud83d\udcdc \ud83c\udf89!\r\n [22 unfixable linting violations found]\r\n```\r\n\r\n```\r\nINSERT OVERWRITE INTO dwh.test_table\r\n\r\nWITH cte1 AS (\r\n\tSELECT *\r\n\tFROM (SELECT\r\n\t\t*,\r\n\t\tROW_NUMBER() OVER (PARTITION BY r ORDER BY updated_at DESC) AS latest\r\n\t\tFROM mongo.temp\r\n\tWHERE latest = 1\r\n))\r\n\r\nSELECT * FROM cte1 WHERE 1=1;\r\n```\r\n\r\nAll of the 22 violations are a mix of L002, L003 and L004.\r\n\r\n### Expected Behaviour\r\n\r\n`sqlfluff` should be able to fix the violations\r\n\r\n### Observed Behaviour\r\n\r\nEven if I try to fix the violations manually, it still shows the same error.\r\n\r\n### How to reproduce\r\n\r\nI will try to generate a sql file that will be able to reproduce the issue\r\n\r\n### Dialect\r\n\r\nSnowflake\r\n\r\n### Version\r\n\r\n1.1.0\r\n\r\n### Configuration\r\n\r\n```\r\n# https://docs.sqlfluff.com/en/stable/rules.html\r\n\r\n[sqlfluff]\r\nexclude_rules = L029, L031, L034\r\n\r\n[sqlfluff:indentation]\r\nindented_joins = true\r\nindented_using_on = true\r\n\r\n[sqlfluff:rules:L002]\r\ntab_space_size = 4\r\n\r\n[sqlfluff:rules:L003]\r\nhanging_indents = true\r\nindent_unit = tab\r\ntab_space_size = 4\r\n\r\n[sqlfluff:rules:L004]\r\nindent_unit = tab\r\ntab_space_size = 4\r\n\r\n[sqlfluff:rules:L010]\r\ncapitalisation_policy = upper\r\n\r\n[sqlfluff:rules:L011]\r\naliasing = explicit\r\n\r\n[sqlfluff:rules:L012]\r\naliasing = explicit\r\n\r\n[sqlfluff:rules:L014]\r\nextended_capitalisation_policy = lower\r\n\r\n[sqlfluff:rules:L016]\r\nignore_comment_clauses = true\r\nignore_comment_lines = true\r\nindent_unit = tab\r\ntab_space_size = 4\r\n\r\n[sqlfluff:rules:L019]\r\ncomma_style = trailing\r\n\r\n[sqlfluff:rules:L022]\r\ncomma_style = trailing\r\n\r\n[sqlfluff:rules:L028]\r\nsingle_table_references = unqualified\r\n\r\n[sqlfluff:rules:L030]\r\nextended_capitalisation_policy = upper\r\n\r\n[sqlfluff:rules:L040]\r\ncapitalisation_policy = upper\r\n\r\n[sqlfluff:rules:L042]\r\nforbid_subquery_in = both\r\n\r\n[sqlfluff:rules:L054]\r\ngroup_by_and_order_by_style = explicit\r\n\r\n[sqlfluff:rules:L063]\r\nextended_capitalisation_policy = upper\r\n\r\n[sqlfluff:rules:L066]\r\nmin_alias_length = 3\r\nmax_alias_length = 15\r\n\r\n[sqlfluff:templater:jinja:context]\r\nparams = {\"DB\": \"DEMO\"}\r\n```\r\n\r\n### Are you willing to work on and submit a PR to address the issue?\r\n\r\n- [X] Yes I am willing to submit a PR!\r\n\r\n### Code of Conduct\r\n\r\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\r\n\n", "hints_text": "Unfortunately there is not much we can do without the SQL that produces this error (ideally a minimal reproducible example SQL) so will need to close this issue if we don\u2019t get that.\nI have updated the issue with a sample query. The query is very vague but it reproduces the error. Let me know if it helps.\nLooks like this simpler example also produces it:\r\n\r\n```sql\r\nWITH cte1 AS (\r\n\tSELECT a\r\n\tFROM (SELECT a)\r\n)\r\n\r\nSELECT a FROM cte1\r\n```\r\n\r\nThis only has one linting failure:\r\n\r\n```\r\n$ sqlfluff lint test.sql --dialect snowflake \r\n== [test.sql] FAIL \r\nL: 3 | P: 7 | L042 | from_expression_element clauses should not contain\r\n | subqueries. Use CTEs instead\r\nAll Finished \ud83d\udcdc \ud83c\udf89!\r\n```\r\n\r\nSo basically L042 gets in a recursive loop when trying to fix CTEs that also break L042.\r\n\r\nFor now you can manually fix that (or exclude L042 for this query) to prevent the error.\nAnother good test query:\r\n```\r\nWITH cte1 AS (\r\n SELECT *\r\n FROM (SELECT * FROM mongo.temp)\r\n)\r\n\r\nSELECT * FROM cte1\r\n```\nPR #3697 avoids the looping behavior. Lint issues are still flagged, but the rule does not attempt to fix it _if_ it would cause a loop. We should still try and figure out why this is happening, so the rule can actually autofix the code, but that's lower priority (and probably a separate PR).", "created_at": "2022-07-31T18:22:12Z", "version": "1.2", "FAIL_TO_PASS": ["test/utils/analysis/test_select_crawler.py::test_select_crawler_constructor[select", "test/utils/analysis/test_select_crawler.py::test_select_crawler_constructor[with", "test/utils/analysis/test_select_crawler.py::test_select_crawler_constructor[\\n", "test/utils/analysis/test_select_crawler.py::test_select_crawler_nested"], "PASS_TO_PASS": [], "environment_setup_commit": "388dd01e05c7dcb880165c7241ed4027d9d0171e"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-3608", "base_commit": "d783e421b714ed989d9e641977ea9b3b6ffaf807", "patch": "diff --git a/src/sqlfluff/cli/__init__.py b/src/sqlfluff/cli/__init__.py\n--- a/src/sqlfluff/cli/__init__.py\n+++ b/src/sqlfluff/cli/__init__.py\n@@ -1 +1,6 @@\n \"\"\"init py for cli.\"\"\"\n+\n+\n+EXIT_SUCCESS = 0\n+EXIT_FAIL = 1\n+EXIT_ERROR = 2\ndiff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py\n--- a/src/sqlfluff/cli/commands.py\n+++ b/src/sqlfluff/cli/commands.py\n@@ -22,6 +22,7 @@\n from tqdm import tqdm\n from sqlfluff.cli.autocomplete import shell_completion_enabled, dialect_shell_complete\n \n+from sqlfluff.cli import EXIT_SUCCESS, EXIT_ERROR, EXIT_FAIL\n from sqlfluff.cli.formatters import (\n format_linting_result_header,\n OutputStreamFormatter,\n@@ -154,7 +155,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):\n Color.red,\n )\n )\n- sys.exit(1)\n+ sys.exit(EXIT_ERROR)\n elif exc_type is SQLFluffUserError:\n click.echo(\n \"\\nUser Error: \"\n@@ -163,7 +164,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):\n Color.red,\n )\n )\n- sys.exit(1)\n+ sys.exit(EXIT_ERROR)\n \n \n def common_options(f: Callable) -> Callable:\n@@ -335,7 +336,7 @@ def get_config(\n color=Color.red,\n )\n )\n- sys.exit(66)\n+ sys.exit(EXIT_ERROR)\n except KeyError:\n click.echo(\n OutputStreamFormatter.colorize_helper(\n@@ -344,7 +345,7 @@ def get_config(\n color=Color.red,\n )\n )\n- sys.exit(66)\n+ sys.exit(EXIT_ERROR)\n from_root_kwargs = {}\n if \"require_dialect\" in kwargs:\n from_root_kwargs[\"require_dialect\"] = kwargs.pop(\"require_dialect\")\n@@ -365,7 +366,7 @@ def get_config(\n color=Color.red,\n )\n )\n- sys.exit(66)\n+ sys.exit(EXIT_ERROR)\n \n \n def get_linter_and_formatter(\n@@ -380,7 +381,7 @@ def get_linter_and_formatter(\n dialect_selector(dialect)\n except KeyError: # pragma: no cover\n click.echo(f\"Error: Unknown dialect '{cfg.get('dialect')}'\")\n- sys.exit(66)\n+ sys.exit(EXIT_ERROR)\n formatter = OutputStreamFormatter(\n output_stream=output_stream or make_output_stream(cfg),\n nocolor=cfg.get(\"nocolor\"),\n@@ -635,7 +636,7 @@ def lint(\n formatter.completion_message()\n sys.exit(result.stats()[\"exit code\"])\n else:\n- sys.exit(0)\n+ sys.exit(EXIT_SUCCESS)\n \n \n def do_fixes(lnt, result, formatter=None, **kwargs):\n@@ -730,7 +731,7 @@ def fix(\n verbose = config.get(\"verbose\")\n progress_bar_configuration.disable_progress_bar = disable_progress_bar\n \n- exit_code = 0\n+ exit_code = EXIT_SUCCESS\n \n formatter.dispatch_config(lnt)\n \n@@ -780,7 +781,7 @@ def fix(\n )\n \n click.echo(stdout, nl=False)\n- sys.exit(1 if templater_error or unfixable_error else exit_code)\n+ sys.exit(EXIT_FAIL if templater_error or unfixable_error else exit_code)\n \n # Lint the paths (not with the fix argument at this stage), outputting as we go.\n click.echo(\"==== finding fixable violations ====\")\n@@ -816,7 +817,7 @@ def fix(\n fixed_file_suffix=fixed_suffix,\n )\n if not success:\n- sys.exit(1) # pragma: no cover\n+ sys.exit(EXIT_FAIL) # pragma: no cover\n else:\n click.echo(\n \"Are you sure you wish to attempt to fix these? [Y/n] \", nl=False\n@@ -833,16 +834,16 @@ def fix(\n fixed_file_suffix=fixed_suffix,\n )\n if not success:\n- sys.exit(1) # pragma: no cover\n+ sys.exit(EXIT_FAIL) # pragma: no cover\n else:\n formatter.completion_message()\n elif c == \"n\":\n click.echo(\"Aborting...\")\n- exit_code = 1\n+ exit_code = EXIT_FAIL\n else: # pragma: no cover\n click.echo(\"Invalid input, please enter 'Y' or 'N'\")\n click.echo(\"Aborting...\")\n- exit_code = 1\n+ exit_code = EXIT_FAIL\n else:\n click.echo(\"==== no fixable linting violations found ====\")\n formatter.completion_message()\n@@ -851,7 +852,7 @@ def fix(\n (\n dict(types=SQLLintError, fixable=False),\n \" [{} unfixable linting violations found]\",\n- 1,\n+ EXIT_FAIL,\n ),\n ]\n for num_violations_kwargs, message_format, error_level in error_types:\n@@ -986,7 +987,7 @@ def parse(\n import cProfile\n except ImportError: # pragma: no cover\n click.echo(\"The cProfiler is not available on your platform.\")\n- sys.exit(1)\n+ sys.exit(EXIT_ERROR)\n pr = cProfile.Profile()\n pr.enable()\n \n@@ -1053,9 +1054,9 @@ def parse(\n click.echo(\"\\n\".join(profiler_buffer.getvalue().split(\"\\n\")[:50]))\n \n if violations_count > 0 and not nofail:\n- sys.exit(66) # pragma: no cover\n+ sys.exit(EXIT_FAIL) # pragma: no cover\n else:\n- sys.exit(0)\n+ sys.exit(EXIT_SUCCESS)\n \n \n # This \"__main__\" handler allows invoking SQLFluff using \"python -m\", which\ndiff --git a/src/sqlfluff/cli/formatters.py b/src/sqlfluff/cli/formatters.py\n--- a/src/sqlfluff/cli/formatters.py\n+++ b/src/sqlfluff/cli/formatters.py\n@@ -6,6 +6,7 @@\n import click\n from colorama import Style\n \n+from sqlfluff.cli import EXIT_FAIL, EXIT_SUCCESS\n from sqlfluff.cli.helpers import (\n get_package_version,\n get_python_version,\n@@ -14,6 +15,7 @@\n wrap_field,\n )\n from sqlfluff.cli.outputstream import OutputStream\n+\n from sqlfluff.core import SQLBaseError, FluffConfig, Linter, TimingSummary\n from sqlfluff.core.enums import Color\n from sqlfluff.core.linter import LintedFile, LintingResult, ParsedString\n@@ -517,7 +519,7 @@ def handle_files_with_tmp_or_prs_errors(self, lint_result: LintingResult) -> int\n color,\n )\n )\n- return 1 if num_filtered_errors else 0\n+ return EXIT_FAIL if num_filtered_errors else EXIT_SUCCESS\n \n def print_out_violations_and_timing(\n self,\ndiff --git a/src/sqlfluff/core/linter/linting_result.py b/src/sqlfluff/core/linter/linting_result.py\n--- a/src/sqlfluff/core/linter/linting_result.py\n+++ b/src/sqlfluff/core/linter/linting_result.py\n@@ -11,6 +11,7 @@\n )\n from typing_extensions import Literal\n \n+from sqlfluff.cli import EXIT_FAIL, EXIT_SUCCESS\n \n from sqlfluff.core.errors import (\n CheckTuple,\n@@ -23,8 +24,6 @@\n \n # Classes needed only for type checking\n from sqlfluff.core.parser.segments.base import BaseSegment\n-\n-\n from sqlfluff.core.linter.linted_dir import LintedDir\n \n \n@@ -133,7 +132,9 @@ def stats(self) -> Dict[str, Any]:\n all_stats[\"unclean rate\"] = 0\n all_stats[\"clean files\"] = all_stats[\"clean\"]\n all_stats[\"unclean files\"] = all_stats[\"unclean\"]\n- all_stats[\"exit code\"] = 65 if all_stats[\"violations\"] > 0 else 0\n+ all_stats[\"exit code\"] = (\n+ EXIT_FAIL if all_stats[\"violations\"] > 0 else EXIT_SUCCESS\n+ )\n all_stats[\"status\"] = \"FAIL\" if all_stats[\"violations\"] > 0 else \"PASS\"\n return all_stats\n \n", "test_patch": "diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py\n--- a/test/cli/commands_test.py\n+++ b/test/cli/commands_test.py\n@@ -71,7 +71,7 @@ def invoke_assert_code(\n def test__cli__command_directed():\n \"\"\"Basic checking of lint functionality.\"\"\"\n result = invoke_assert_code(\n- ret_code=65,\n+ ret_code=1,\n args=[\n lint,\n [\n@@ -95,7 +95,7 @@ def test__cli__command_dialect():\n \"\"\"Check the script raises the right exception on an unknown dialect.\"\"\"\n # The dialect is unknown should be a non-zero exit code\n invoke_assert_code(\n- ret_code=66,\n+ ret_code=2,\n args=[\n lint,\n [\n@@ -112,7 +112,7 @@ def test__cli__command_no_dialect():\n \"\"\"Check the script raises the right exception no dialect.\"\"\"\n # The dialect is unknown should be a non-zero exit code\n result = invoke_assert_code(\n- ret_code=1,\n+ ret_code=2,\n args=[\n lint,\n [\"-\"],\n@@ -129,7 +129,7 @@ def test__cli__command_parse_error_dialect_explicit_warning():\n # and a human-readable warning should be dislayed.\n # Dialect specified as commandline option.\n result = invoke_assert_code(\n- ret_code=66,\n+ ret_code=1,\n args=[\n parse,\n [\n@@ -152,7 +152,7 @@ def test__cli__command_parse_error_dialect_implicit_warning():\n # and a human-readable warning should be dislayed.\n # Dialect specified in .sqlfluff config.\n result = invoke_assert_code(\n- ret_code=66,\n+ ret_code=1,\n args=[\n # Config sets dialect to tsql\n parse,\n@@ -173,7 +173,7 @@ def test__cli__command_parse_error_dialect_implicit_warning():\n def test__cli__command_dialect_legacy():\n \"\"\"Check the script raises the right exception on a legacy dialect.\"\"\"\n result = invoke_assert_code(\n- ret_code=66,\n+ ret_code=2,\n args=[\n lint,\n [\n@@ -190,7 +190,7 @@ def test__cli__command_dialect_legacy():\n def test__cli__command_extra_config_fail():\n \"\"\"Check the script raises the right exception non-existent extra config path.\"\"\"\n result = invoke_assert_code(\n- ret_code=66,\n+ ret_code=2,\n args=[\n lint,\n [\n@@ -429,7 +429,7 @@ def test__cli__command_lint_parse(command):\n [\"test/fixtures/cli/unknown_jinja_tag/test.sql\", \"-vvvvvvv\"],\n \"y\",\n ),\n- 65,\n+ 1,\n ),\n ],\n )\n@@ -461,7 +461,7 @@ def test__cli__command_lint_skip_ignore_files():\n \"--disregard-sqlfluffignores\",\n ],\n )\n- assert result.exit_code == 65\n+ assert result.exit_code == 1\n assert \"L009\" in result.output.strip()\n \n \n@@ -488,7 +488,7 @@ def test__cli__command_lint_ignore_local_config():\n \"test/fixtures/cli/ignore_local_config/ignore_local_config_test.sql\",\n ],\n )\n- assert result.exit_code == 65\n+ assert result.exit_code == 1\n assert \"L012\" in result.output.strip()\n \n \n@@ -561,7 +561,7 @@ def generic_roundtrip_test(\n old_mode = stat.S_IMODE(status.st_mode)\n # Check that we first detect the issue\n invoke_assert_code(\n- ret_code=65, args=[lint, [\"--dialect=ansi\", \"--rules\", rulestring, filepath]]\n+ ret_code=1, args=[lint, [\"--dialect=ansi\", \"--rules\", rulestring, filepath]]\n )\n # Fix the file (in force mode)\n if force:\n@@ -997,7 +997,7 @@ def test__cli__command_fix_stdin_error_exit_code(\n \"rule,fname,prompt,exit_code,fix_exit_code\",\n [\n (\"L001\", \"test/fixtures/linter/indentation_errors.sql\", \"y\", 0, 0),\n- (\"L001\", \"test/fixtures/linter/indentation_errors.sql\", \"n\", 65, 1),\n+ (\"L001\", \"test/fixtures/linter/indentation_errors.sql\", \"n\", 1, 1),\n ],\n )\n def test__cli__command__fix_no_force(rule, fname, prompt, exit_code, fix_exit_code):\n@@ -1075,7 +1075,7 @@ def test__cli__command_parse_serialize_from_stdin(serialize, write_file, tmp_pat\n ],\n }\n ],\n- 65,\n+ 1,\n ),\n ],\n )\n@@ -1115,7 +1115,7 @@ def test__cli__command_lint_serialize_from_stdin(serialize, sql, expected, exit_\n )\n def test__cli__command_fail_nice_not_found(command):\n \"\"\"Check commands fail as expected when then don't find files.\"\"\"\n- result = invoke_assert_code(args=command, ret_code=1)\n+ result = invoke_assert_code(args=command, ret_code=2)\n assert \"could not be accessed\" in result.output\n \n \n@@ -1180,7 +1180,7 @@ def test__cli__command_lint_serialize_multiple_files(serialize, write_file, tmp_\n # note the file is in here twice. two files = two payloads.\n result = invoke_assert_code(\n args=[lint, cmd_args],\n- ret_code=65,\n+ ret_code=1,\n )\n \n if write_file:\n@@ -1226,7 +1226,7 @@ def test__cli__command_lint_serialize_github_annotation():\n \"--disable_progress_bar\",\n ),\n ],\n- ret_code=65,\n+ ret_code=1,\n )\n result = json.loads(result.output)\n assert result == [\n@@ -1337,7 +1337,7 @@ def test__cli__command_lint_serialize_github_annotation_native():\n \"--disable_progress_bar\",\n ),\n ],\n- ret_code=65,\n+ ret_code=1,\n )\n \n assert result.output == \"\\n\".join(\n@@ -1381,7 +1381,7 @@ def test__cli__command_lint_serialize_annotation_level_error_failure_equivalent(\n \"--disable_progress_bar\",\n ),\n ],\n- ret_code=65,\n+ ret_code=1,\n )\n \n result_failure = invoke_assert_code(\n@@ -1396,7 +1396,7 @@ def test__cli__command_lint_serialize_annotation_level_error_failure_equivalent(\n \"--disable_progress_bar\",\n ),\n ],\n- ret_code=65,\n+ ret_code=1,\n )\n \n assert result_error.output == result_failure.output\n@@ -1450,7 +1450,7 @@ def test_cli_encoding(encoding, method, expect_success, tmpdir):\n shutil.copy(sql_path, tmpdir)\n options = [str(tmpdir / \"encoding_test.sql\")]\n result = invoke_assert_code(\n- ret_code=65,\n+ ret_code=1,\n args=[\n lint,\n options,\n@@ -1479,7 +1479,7 @@ def test_cli_no_disable_noqa_flag():\n def test_cli_disable_noqa_flag():\n \"\"\"Test that --disable_noqa flag ignores inline noqa comments.\"\"\"\n result = invoke_assert_code(\n- ret_code=65,\n+ ret_code=1,\n args=[\n lint,\n [\n@@ -1563,7 +1563,7 @@ def test_cli_lint_enabled_progress_bar_multiple_paths(\n ) -> None:\n \"\"\"When progress bar is enabled, there should be some tracks in output.\"\"\"\n result = invoke_assert_code(\n- ret_code=65,\n+ ret_code=1,\n args=[\n lint,\n [\ndiff --git a/test/rules/std_roundtrip_test.py b/test/rules/std_roundtrip_test.py\n--- a/test/rules/std_roundtrip_test.py\n+++ b/test/rules/std_roundtrip_test.py\n@@ -34,7 +34,7 @@ def generic_roundtrip_test(source_file, rulestring):\n runner = CliRunner()\n # Check that we first detect the issue\n result = runner.invoke(lint, [\"--rules\", rulestring, \"--dialect=ansi\", filepath])\n- assert result.exit_code == 65\n+ assert result.exit_code == 1\n # Fix the file (in force mode)\n result = runner.invoke(\n fix, [\"--rules\", rulestring, \"--dialect=ansi\", \"-f\", filepath]\n@@ -80,7 +80,7 @@ def jinja_roundtrip_test(\n result = runner.invoke(\n lint, [\"--rules\", rulestring, \"--dialect=ansi\", sql_filepath]\n )\n- assert result.exit_code == 65\n+ assert result.exit_code == 1\n # Fix the file (in force mode)\n result = runner.invoke(\n fix, [\"--rules\", rulestring, \"-f\", \"--dialect=ansi\", sql_filepath]\n", "problem_statement": "Return codes are inconsistent\n### Search before asking\r\n\r\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\r\n\r\n\r\n### What Happened\r\n\r\nWorking on #3431 - I noticed that we're inconsistent in our return codes.\r\n\r\nIn `commands.py` we call `sys.exit()` in 15 places (currently).\r\n\r\n- Twice we call `sys.exit(0)` on success, at the end of `parse` and `lint` (`fix` is a handled differently, see below). \u2714\ufe0f \r\n- Six times we call `sys.exit(1)` for a selection of things:\r\n - Not having `cProfiler` installed.\r\n - Failing to apply fixes\r\n - User Errors and OSError (in `PathAndUserErrorHandler`)\r\n- Five times we call `sys.exit(66)` for a selection of things:\r\n - User Errors (including unknown dialect or failing to load a dialect or config)\r\n - If parsing failed when calling `parse`.\r\n- Once we use `handle_files_with_tmp_or_prs_errors` to determine the exit code (which returns 1 or 0)\r\n- Once we use `LintingResult.stats` to determine the exit code (which returns either 65 or 0)\r\n- Once we do a mixture of the above (see end of `fix`)\r\n\r\nThis neither DRY, or consistent ... or helpful?\r\n\r\n### Expected Behaviour\r\n\r\nWe should have consistent return codes for specific scenarios. There are up for discussion, but I would suggest:\r\n\r\n- 0 for success (obviously)\r\n- 1 for a fail which is error related: not having libraries installed, user errors etc...\r\n- 65 for a linting fail (i.e. no errors in running, but issues were found in either parsing or linting).\r\n- 66 for a fixing fail (i.e. we tried to fix errors but failed to do so for some reason).\r\n\r\nThese would be defined as constants at the top of `commands.py`.\r\n\r\n### Observed Behaviour\r\n\r\nsee above\r\n\r\n### How to reproduce\r\n\r\nsee above\r\n\r\n### Dialect\r\n\r\nN/A\r\n\r\n### Version\r\n\r\nDescription is as per code in #3431\r\n\r\n### Configuration\r\n\r\n-\r\n\r\n### Are you willing to work on and submit a PR to address the issue?\r\n\r\n- [X] Yes I am willing to submit a PR!\r\n\r\n### Code of Conduct\r\n\r\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\r\n\n", "hints_text": "I'm happy to contribute the changes for this one, but would appreciate views on what the error codes we should align on first @barrywhart @tunetheweb \nI'm not familiar with any widespread conventions about exit codes, except to keep them below 256.\r\n\r\nThis Stack Overflow post has a lot of discussion, often self-contradictory. https://stackoverflow.com/questions/1101957/are-there-any-standard-exit-status-codes-in-linux\r\n\r\nOverall, your proposal sounds good to me.\r\n\r\nCan you also search the existing issues for any mention of exit codes? I think there may be one or two open issues, perhaps related to the behavior when \"fix\" finds issues but some are unfixable. Because of its multifaceted nature as a linter and fixer that is used both interactively (e.g. during pre-commit) and in batch (CICD), SQLFluff perhaps has more stringent requirements for precise exit codes than some other tools. Do you think it'd be useful to review existing (and or write some new) user documentation before starting the coding, to help get a better understanding of the various use cases?\r\n\r\n\r\n\nAgree with @barrywhart 's comments.\r\n\r\nOnly question is why 65/66 instead of just 2/3?\n> Only question is why 65/66 instead of just 2/3?\r\n\r\nThis was initially because I had read that codes 0-64 were reserved for system usage but it appears things aren't that consistent.\r\n\r\n> This Stack Overflow post has a lot of discussion, often self-contradictory...\r\n\r\nI'm wondering based on this post whether we should simplify things:\r\n- 0: success\r\n- 1: fail (on linting or fixing, but due to finding issues with code or unable to fix, no \"errors\")\r\n- 2: fail because misuse or error\r\n\r\nIt's slightly less granular but a little more consistent with the bash approach (from the most recent post on that SO question):\r\n\r\n> Exit status 0: success\r\n> Exit status 1: \"failure\", as defined by the program\r\n> Exit status 2: command line usage error\nCleaning up the exit codes seems sensible. How likely do we think it is to break things for users?\nRelatively unlikely I reckon - I'm not sure the existing codes are sufficiently granular to be useful right now.", "created_at": "2022-07-14T15:06:34Z", "version": "1.1", "FAIL_TO_PASS": ["test/cli/commands_test.py::test__cli__command_directed", "test/cli/commands_test.py::test__cli__command_dialect", "test/cli/commands_test.py::test__cli__command_no_dialect", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_explicit_warning", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_implicit_warning", "test/cli/commands_test.py::test__cli__command_dialect_legacy", "test/cli/commands_test.py::test__cli__command_extra_config_fail", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command3-1]", "test/cli/commands_test.py::test__cli__command_lint_skip_ignore_files", "test/cli/commands_test.py::test__cli__command_lint_ignore_local_config", "test/cli/commands_test.py::test__cli__command__fix[L001-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/whitespace_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L003-test/fixtures/linter/indentation_error_hard.sql]", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-y-0-0]", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-n-1-1]", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[SElect", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command0]", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command1]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation_native", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation-native]", "test/cli/commands_test.py::test_encoding[utf-8-ascii]", "test/cli/commands_test.py::test_encoding[utf-8-sig-UTF-8-SIG]", "test/cli/commands_test.py::test_encoding[utf-32-UTF-32]", "test/cli/commands_test.py::test_cli_encoding[utf-8-command-line-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-command-line-True]", "test/cli/commands_test.py::test_cli_encoding[utf-8-config-file-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-config-file-True]", "test/cli/commands_test.py::test_cli_disable_noqa_flag", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_paths", "test/rules/std_roundtrip_test.py::test__cli__command__fix[L001-test/fixtures/linter/indentation_errors.sql]", "test/rules/std_roundtrip_test.py::test__cli__command__fix[L008-test/fixtures/linter/whitespace_errors.sql]", "test/rules/std_roundtrip_test.py::test__cli__command__fix[L008-test/fixtures/linter/indentation_errors.sql]", "test/rules/std_roundtrip_test.py::test__cli__command__fix[L010-test/fixtures/linter/whitespace_errors.sql]", "test/rules/std_roundtrip_test.py::test__cli__command__fix[L011-test/fixtures/dialects/ansi/select_simple_i.sql]", "test/rules/std_roundtrip_test.py::test__cli__command__fix[L012-test/fixtures/dialects/ansi/select_simple_i.sql]", "test/rules/std_roundtrip_test.py::test__cli__command__fix_templated[L010]", "test/rules/std_roundtrip_test.py::test__cli__command__fix_templated[L001]"], "PASS_TO_PASS": ["test/cli/commands_test.py::test__cli__command_lint_stdin[command0]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command1]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command2]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command0]", "test/cli/commands_test.py::test__cli__command_lint_parse[command1]", "test/cli/commands_test.py::test__cli__command_lint_parse[command2]", "test/cli/commands_test.py::test__cli__command_lint_parse[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command4]", "test/cli/commands_test.py::test__cli__command_lint_parse[command5]", "test/cli/commands_test.py::test__cli__command_lint_parse[command6]", "test/cli/commands_test.py::test__cli__command_lint_parse[command7]", "test/cli/commands_test.py::test__cli__command_lint_parse[command8]", "test/cli/commands_test.py::test__cli__command_lint_parse[command9]", "test/cli/commands_test.py::test__cli__command_lint_parse[command10]", "test/cli/commands_test.py::test__cli__command_lint_parse[command11]", "test/cli/commands_test.py::test__cli__command_lint_parse[command12]", "test/cli/commands_test.py::test__cli__command_lint_parse[command13]", "test/cli/commands_test.py::test__cli__command_lint_parse[command14]", "test/cli/commands_test.py::test__cli__command_lint_parse[command15]", "test/cli/commands_test.py::test__cli__command_lint_parse[command16]", "test/cli/commands_test.py::test__cli__command_lint_parse[command17]", "test/cli/commands_test.py::test__cli__command_lint_parse[command18]", "test/cli/commands_test.py::test__cli__command_lint_parse[command19]", "test/cli/commands_test.py::test__cli__command_lint_parse[command20]", "test/cli/commands_test.py::test__cli__command_lint_parse[command21]", "test/cli/commands_test.py::test__cli__command_lint_parse[command22]", "test/cli/commands_test.py::test__cli__command_lint_parse[command23]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command0-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command1-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command2-1]", "test/cli/commands_test.py::test__cli__command_lint_warning_explicit_file_ignored", "test/cli/commands_test.py::test__cli__command_versioning", "test/cli/commands_test.py::test__cli__command_version", "test/cli/commands_test.py::test__cli__command_rules", "test/cli/commands_test.py::test__cli__command_dialects", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_templating_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error_FIX_EVEN_UNPARSABLE]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[2_files_with_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-True]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-True]", "test/cli/commands_test.py::test__cli__fix_loop_limit_behavior[--", "test/cli/commands_test.py::test__cli__command_fix_stdin[select", "test/cli/commands_test.py::test__cli__command_fix_stdin[", "test/cli/commands_test.py::test__cli__command_fix_stdin[SELECT", "test/cli/commands_test.py::test__cli__command_fix_stdin_logging_to_stderr", "test/cli/commands_test.py::test__cli__command_fix_stdin_safety", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[create", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[select", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-json]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[select", "test/cli/commands_test.py::test__cli__command_lint_nocolor", "test/cli/commands_test.py::test___main___help", "test/cli/commands_test.py::test_cli_no_disable_noqa_flag", "test/cli/commands_test.py::test_cli_get_default_config", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_files"], "environment_setup_commit": "d83ab36bbb21f62cf0780d095a8be8cd366735d7"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-3435", "base_commit": "8e724ef8906eecce7179f4b7c52d4fc0672e4bd9", "patch": "diff --git a/src/sqlfluff/rules/L027.py b/src/sqlfluff/rules/L027.py\n--- a/src/sqlfluff/rules/L027.py\n+++ b/src/sqlfluff/rules/L027.py\n@@ -99,23 +99,6 @@ def _lint_references_and_aliases(\n )\n )\n \n- all_table_aliases = [t.ref_str for t in table_aliases] + standalone_aliases\n-\n- # For qualified references, we want to check that the alias is actually\n- # valid\n- if (\n- this_ref_type == \"qualified\"\n- and list(r.iter_raw_references())[0].part not in all_table_aliases\n- ):\n- violation_buff.append(\n- LintResult(\n- anchor=r,\n- description=f\"Qualified reference {r.raw!r} not found in \"\n- f\"available tables/view aliases {all_table_aliases} in select \"\n- \"with more than one referenced table/view.\",\n- )\n- )\n-\n return violation_buff or None\n \n def _init_ignore_words_list(self):\n", "test_patch": "diff --git a/test/fixtures/rules/std_rule_cases/L027.yml b/test/fixtures/rules/std_rule_cases/L027.yml\n--- a/test/fixtures/rules/std_rule_cases/L027.yml\n+++ b/test/fixtures/rules/std_rule_cases/L027.yml\n@@ -221,43 +221,6 @@ test_pass_rowtype_with_join:\n core:\n dialect: hive\n \n-test_fail_column_name_not_found_in_table_aliases_bigquery:\n- # qualified reference should actually exists in table aliases\n- fail_str: |\n- SELECT\n- a.bar,\n- b.foo,\n- this_is.some_struct.id\n- FROM\n- a LEFT JOIN b ON TRUE\n- configs:\n- core:\n- dialect: bigquery\n-\n-test_pass_column_name_is_a_struct_bigquery:\n- # check structs work as expected\n- pass_str: |\n- SELECT\n- a.bar,\n- b.this_is.some_struct.id\n- FROM\n- a LEFT JOIN b ON TRUE\n- configs:\n- core:\n- dialect: bigquery\n-\n-test_pass_column_name_from_unnest_bigquery:\n- # Check that we allow an table alias come from UNNEST statement\n- pass_str: |\n- SELECT\n- a.bar,\n- e.foo\n- FROM\n- a LEFT JOIN UNEST(a.events) AS e\n- configs:\n- core:\n- dialect: bigquery\n-\n test_fail_table_plus_flatten_snowflake_1:\n # FLATTEN() returns a table, thus there are two tables, thus lint failure.\n fail_str: |\n@@ -328,3 +291,48 @@ test_pass_ignore_words_regex_bigquery_declare_example:\n rules:\n L027:\n ignore_words_regex: ^_\n+\n+test_pass_redshift:\n+ # This was failing in issue 3380.\n+ pass_str:\n+ SELECT account.id\n+ FROM salesforce_sd.account\n+ INNER JOIN salesforce_sd.\"user\" ON salesforce_sd.\"user\".id = account.ownerid\n+ configs:\n+ core:\n+ dialect: redshift\n+\n+test_pass_tsql:\n+ # This was failing in issue 3342.\n+ pass_str:\n+ select\n+ psc.col1\n+ from\n+ tbl1 as psc\n+ where\n+ exists\n+ (\n+ select 1 as data\n+ from\n+ tbl2 as pr\n+ join tbl2 as c on c.cid = pr.cid\n+ where\n+ c.col1 = 'x'\n+ and pr.col2 <= convert(date, getdate())\n+ and pr.pid = psc.pid\n+ )\n+ configs:\n+ core:\n+ dialect: tsql\n+\n+test_pass_ansi:\n+ # This was failing in issue 3055.\n+ pass_str: |\n+ SELECT my_col\n+ FROM my_table\n+ WHERE EXISTS (\n+ SELECT 1\n+ FROM other_table\n+ INNER JOIN mapping_table ON (mapping_table.other_fk = other_table.id_pk)\n+ WHERE mapping_table.kind = my_table.kind\n+ )\ndiff --git a/test/rules/std_test.py b/test/rules/std_test.py\n--- a/test/rules/std_test.py\n+++ b/test/rules/std_test.py\n@@ -68,7 +68,7 @@\n ),\n (\"L016\", \"block_comment_errors_2.sql\", [(1, 85), (2, 86)]),\n # Column references\n- (\"L027\", \"column_references.sql\", [(1, 8), (1, 11)]),\n+ (\"L027\", \"column_references.sql\", [(1, 8)]),\n (\"L027\", \"column_references_bare_function.sql\", []),\n (\"L026\", \"column_references.sql\", [(1, 11)]),\n (\"L025\", \"column_references.sql\", [(2, 11)]),\n", "problem_statement": "L027: outer-level table not found in WHERE clause sub-select\n### Search before asking\r\n\r\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\r\n\r\n\r\n### What Happened\r\n\r\nOuter-level table/view referenced in sub-select inside `WHERE` clause is not being detected.\r\n\r\nThis error seems to only occur when the sub-select contains joins.\r\n\r\n### Expected Behaviour\r\n\r\nNo error\r\n\r\n### Observed Behaviour\r\n\r\n```\r\nL: 7 | P: 32 | L027 | Qualified reference 'my_table.kind' not found in\r\n | available tables/view aliases ['other_table',\r\n | 'mapping_table'] in select with more than one referenced\r\n | table/view.\r\n```\r\n\r\n### How to reproduce\r\n\r\n```sql\r\nSELECT my_col\r\nFROM my_table\r\nWHERE EXISTS (\r\n SELECT 1\r\n FROM other_table\r\n INNER JOIN mapping_table ON (mapping_table.other_fk = other_table.id_pk)\r\n WHERE mapping_table.kind = my_table.kind\r\n);\r\n```\r\n\r\n### Dialect\r\n\r\npostgres\r\n\r\n### Version\r\n\r\nsqlfluff, version 0.12.0\r\n\r\n### Configuration\r\n\r\n```\r\n[sqlfluff]\r\nnocolor = True\r\ndialect = postgres\r\n```\r\n\r\n### Are you willing to work on and submit a PR to address the issue?\r\n\r\n- [ ] Yes I am willing to submit a PR!\r\n\r\n### Code of Conduct\r\n\r\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\r\n\n", "hints_text": "Does L027 use `SelectCrawler`? This sounds like an issue where it may be helpful.\r\n\r\nRelated: Rules that use `SelectCrawler` may be good candidates to benefit from setting `recurse_into` to `False`. (Setting the flag is just a start. This also requires reworking the rule code, hopefully no more than 1-2 hours of work.)\nAnswering my own question: It does not seem to use `SelectCrawler`. Rules that currently use it:\r\n* L025\r\n* L026\r\n* L044\r\n* L045\r\n\r\nFrom a quick look at the YML test files for each of these rules, I suggest L044 would be the best one to review in terms of handling similar requirements. Look for test cases that mention \"subquery\".\nI think a very similar fix to that implemented in this [PR for L028](https://github.com/sqlfluff/sqlfluff/pull/3156) will also work here. In particular, notice the code that looks at `query.parent` to find tables that are \"visible\" to a particular query.\r\n\r\nhttps://github.com/sqlfluff/sqlfluff/blob/main/src/sqlfluff/rules/L028.py#L108L114\nRelated to #3380, possibly duplicate", "created_at": "2022-06-07T18:47:03Z", "version": "0.13", "FAIL_TO_PASS": ["test/rules/std_test.py::test__rules__std_file[L027-column_references.sql-violations16]"], "PASS_TO_PASS": ["test/rules/std_test.py::test__rules__std_file[L001-indentation_errors.sql-violations0]", "test/rules/std_test.py::test__rules__std_file[L002-indentation_errors.sql-violations1]", "test/rules/std_test.py::test__rules__std_file[L003-indentation_errors.sql-violations2]", "test/rules/std_test.py::test__rules__std_file[L004-indentation_errors.sql-violations3]", "test/rules/std_test.py::test__rules__std_file[L005-whitespace_errors.sql-violations4]", "test/rules/std_test.py::test__rules__std_file[L019-whitespace_errors.sql-violations5]", "test/rules/std_test.py::test__rules__std_file[L008-whitespace_errors.sql-violations6]", "test/rules/std_test.py::test__rules__std_file[L006-operator_errors.sql-violations7]", "test/rules/std_test.py::test__rules__std_file[L039-operator_errors.sql-violations8]", "test/rules/std_test.py::test__rules__std_file[L007-operator_errors.sql-violations9]", "test/rules/std_test.py::test__rules__std_file[L006-operator_errors_negative.sql-violations10]", "test/rules/std_test.py::test__rules__std_file[L039-operator_errors_negative.sql-violations11]", "test/rules/std_test.py::test__rules__std_file[L003-indentation_error_hard.sql-violations12]", "test/rules/std_test.py::test__rules__std_file[L003-indentation_error_contained.sql-violations13]", "test/rules/std_test.py::test__rules__std_file[L016-block_comment_errors.sql-violations14]", "test/rules/std_test.py::test__rules__std_file[L016-block_comment_errors_2.sql-violations15]", "test/rules/std_test.py::test__rules__std_file[L027-column_references_bare_function.sql-violations17]", "test/rules/std_test.py::test__rules__std_file[L026-column_references.sql-violations18]", "test/rules/std_test.py::test__rules__std_file[L025-column_references.sql-violations19]", "test/rules/std_test.py::test__rules__std_file[L021-select_distinct_group_by.sql-violations20]", "test/rules/std_test.py::test__rules__std_file[L006-operator_errors_ignore.sql-violations21]", "test/rules/std_test.py::test__rules__std_file[L031-aliases_in_join_error.sql-violations22]", "test/rules/std_test.py::test__rules__std_file[L046-heavy_templating.sql-violations23]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict0]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict1]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict2]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict3]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict4]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict5]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict6]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict7]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict8]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict9]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict10]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict11]"], "environment_setup_commit": "6e8ce43a4958dbaa56256365c2a89d8db92e07d6"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-3904", "base_commit": "2ac8e125604aa1f19d9811f7dc5bd56eefa654ac", "patch": "diff --git a/src/sqlfluff/cli/click_deprecated_option.py b/src/sqlfluff/cli/click_deprecated_option.py\nnew file mode 100644\n--- /dev/null\n+++ b/src/sqlfluff/cli/click_deprecated_option.py\n@@ -0,0 +1,104 @@\n+\"\"\"Allows to provide deprecated options for click's command.\"\"\"\n+\n+from typing import Any, Callable\n+\n+import click\n+from click import Context, OptionParser, echo, style\n+from click.parser import Option, ParsingState\n+\n+\n+class DeprecatedOption(click.Option):\n+ \"\"\"Allows to provide deprecated options for click's command.\n+\n+ Works with `DeprecatedOptionsCommand` (see below).\n+ Expects to be provided into standard `@click.option` with:\n+ * two parameter declarations arguments - old one (deprecated)\n+ and new one (preferred);\n+ * `cls` parameter (standard click Option) as `cls=DeprecatedOption`;\n+ * `deprecated` parameter - which says which ones are deprecated,\n+ like`deprecated=[\"--disable_progress_bar\"]1.\n+\n+ This is based on\n+ * https://stackoverflow.com/a/50402799/5172513\n+\n+ It's somewhat hackish and may broke when click internals are changed, it is even\n+ mentioned in SO:\n+ > This code reaches into some private structures in the parser, but this is\n+ unlikely to be an issue. This parser code was last changed 4 years ago.\n+ The parser code is unlikely to undergo significant revisions.\n+\n+ Hopefully will be removed when\n+ * https://github.com/pallets/click/issues/2263\n+ is finished.\n+ \"\"\"\n+\n+ def __init__(self, *args, **kwargs):\n+ self.deprecated = kwargs.pop(\"deprecated\", ())\n+ self.preferred = args[0][-1]\n+\n+ super().__init__(*args, **kwargs)\n+\n+\n+class DeprecatedOptionsCommand(click.Command):\n+ \"\"\"Allows to provide deprecated options for click's command.\n+\n+ Works with `DeprecatedOption` (see above).\n+ Expects to be provided into standard `@click.command` as:\n+ * `@cli.command(cls=DeprecatedOptionsCommand)`\n+ \"\"\"\n+\n+ def make_parser(self, ctx: Context) -> OptionParser:\n+ \"\"\"Hook 'make_parser' and during processing check the name.\n+\n+ Used to invoke the option to see if it is preferred.\n+ \"\"\"\n+ parser: OptionParser = super().make_parser(ctx)\n+\n+ # get the parser options\n+ options = set(parser._short_opt.values())\n+ options |= set(parser._long_opt.values())\n+\n+ for option in options:\n+ if not isinstance(option.obj, DeprecatedOption):\n+ continue\n+\n+ option.process = self._make_process(option) # type: ignore\n+\n+ return parser\n+\n+ def _make_process(self, an_option: Option) -> Callable:\n+ \"\"\"Construct a closure to the parser option processor.\"\"\"\n+ orig_process: Callable = an_option.process\n+ deprecated = getattr(an_option.obj, \"deprecated\", None)\n+ preferred = getattr(an_option.obj, \"preferred\", None)\n+\n+ if not deprecated:\n+ raise ValueError(\n+ f\"Expected `deprecated` value for `{an_option.obj.name!r}`\"\n+ )\n+\n+ def process(value: Any, state: ParsingState) -> None:\n+ \"\"\"Custom process method.\n+\n+ The function above us on the stack used 'opt' to\n+ pick option from a dict, see if it is deprecated.\n+ \"\"\"\n+ # reach up the stack and get 'opt'\n+ import inspect\n+\n+ frame = inspect.currentframe()\n+ try:\n+ opt = frame.f_back.f_locals.get(\"opt\") # type: ignore\n+ finally:\n+ del frame\n+\n+ if opt in deprecated: # type: ignore\n+ msg = (\n+ f\"DeprecationWarning: The option {opt!r} is deprecated, \"\n+ f\"use {preferred!r}.\"\n+ )\n+ echo(style(msg, fg=\"red\"), err=True)\n+\n+ return orig_process(value, state)\n+\n+ return process\ndiff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py\n--- a/src/sqlfluff/cli/commands.py\n+++ b/src/sqlfluff/cli/commands.py\n@@ -23,6 +23,10 @@\n from sqlfluff.cli.autocomplete import shell_completion_enabled, dialect_shell_complete\n \n from sqlfluff.cli import EXIT_SUCCESS, EXIT_ERROR, EXIT_FAIL\n+from sqlfluff.cli.click_deprecated_option import (\n+ DeprecatedOption,\n+ DeprecatedOptionsCommand,\n+)\n from sqlfluff.cli.formatters import (\n format_linting_result_header,\n OutputStreamFormatter,\n@@ -455,7 +459,7 @@ def dump_file_payload(filename: Optional[str], payload: str):\n click.echo(payload)\n \n \n-@cli.command()\n+@cli.command(cls=DeprecatedOptionsCommand)\n @common_options\n @core_options\n @click.option(\n@@ -509,8 +513,11 @@ def dump_file_payload(filename: Optional[str], payload: str):\n )\n @click.option(\n \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n is_flag=True,\n help=\"Disables progress bars.\",\n+ cls=DeprecatedOption,\n+ deprecated=[\"--disable_progress_bar\"],\n )\n @click.argument(\"paths\", nargs=-1, type=click.Path(allow_dash=True))\n def lint(\n@@ -704,7 +711,7 @@ def do_fixes(lnt, result, formatter=None, **kwargs):\n ),\n )\n @click.option(\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n is_flag=True,\n help=\"Disables progress bars.\",\n )\n", "test_patch": "diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py\n--- a/test/cli/commands_test.py\n+++ b/test/cli/commands_test.py\n@@ -112,7 +112,7 @@ def test__cli__command_directed():\n args=[\n lint,\n [\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n \"test/fixtures/linter/indentation_error_simple.sql\",\n ],\n ],\n@@ -1002,7 +1002,7 @@ def test__cli__command_fix_stdin(stdin, rules, stdout):\n result = invoke_assert_code(\n args=[\n fix,\n- (\"-\", \"--rules\", rules, \"--disable_progress_bar\", \"--dialect=ansi\"),\n+ (\"-\", \"--rules\", rules, \"--disable-progress-bar\", \"--dialect=ansi\"),\n ],\n cli_input=stdin,\n )\n@@ -1036,7 +1036,7 @@ def test__cli__command_fix_stdin_safety():\n \n # just prints the very same thing\n result = invoke_assert_code(\n- args=[fix, (\"-\", \"--disable_progress_bar\", \"--dialect=ansi\")],\n+ args=[fix, (\"-\", \"--disable-progress-bar\", \"--dialect=ansi\")],\n cli_input=perfect_sql,\n )\n assert result.output.strip() == perfect_sql\n@@ -1177,7 +1177,7 @@ def test__cli__command_lint_serialize_from_stdin(serialize, sql, expected, exit_\n \"L010\",\n \"--format\",\n serialize,\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n \"--dialect=ansi\",\n ),\n ],\n@@ -1222,7 +1222,7 @@ def test__cli__command_lint_nocolor(isatty, should_strip_ansi, capsys, tmpdir):\n \"--nocolor\",\n \"--dialect\",\n \"ansi\",\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n fpath,\n \"--write-output\",\n output_file,\n@@ -1253,7 +1253,7 @@ def test__cli__command_lint_serialize_multiple_files(serialize, write_file, tmp_\n fpath,\n \"--format\",\n serialize,\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n )\n \n if write_file:\n@@ -1310,7 +1310,7 @@ def test__cli__command_lint_serialize_github_annotation():\n \"github-annotation\",\n \"--annotation-level\",\n \"warning\",\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n ),\n ],\n ret_code=1,\n@@ -1421,7 +1421,7 @@ def test__cli__command_lint_serialize_github_annotation_native():\n \"github-annotation-native\",\n \"--annotation-level\",\n \"error\",\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n ),\n ],\n ret_code=1,\n@@ -1465,7 +1465,7 @@ def test__cli__command_lint_serialize_annotation_level_error_failure_equivalent(\n serialize,\n \"--annotation-level\",\n \"error\",\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n ),\n ],\n ret_code=1,\n@@ -1480,7 +1480,7 @@ def test__cli__command_lint_serialize_annotation_level_error_failure_equivalent(\n serialize,\n \"--annotation-level\",\n \"failure\",\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n ),\n ],\n ret_code=1,\n@@ -1613,6 +1613,25 @@ def test_cli_lint_disabled_progress_bar(\n self, mock_disable_progress_bar: MagicMock\n ) -> None:\n \"\"\"When progress bar is disabled, nothing should be printed into output.\"\"\"\n+ result = invoke_assert_code(\n+ args=[\n+ lint,\n+ [\n+ \"--disable-progress-bar\",\n+ \"test/fixtures/linter/passing.sql\",\n+ ],\n+ ],\n+ )\n+ raw_output = repr(result.output)\n+\n+ assert \"\\rpath test/fixtures/linter/passing.sql:\" not in raw_output\n+ assert \"\\rparsing: 0it\" not in raw_output\n+ assert \"\\r\\rlint by rules:\" not in raw_output\n+\n+ def test_cli_lint_disabled_progress_bar_deprecated_option(\n+ self, mock_disable_progress_bar: MagicMock\n+ ) -> None:\n+ \"\"\"Same as above but checks additionally if deprecation warning is printed.\"\"\"\n result = invoke_assert_code(\n args=[\n lint,\n@@ -1627,6 +1646,10 @@ def test_cli_lint_disabled_progress_bar(\n assert \"\\rpath test/fixtures/linter/passing.sql:\" not in raw_output\n assert \"\\rparsing: 0it\" not in raw_output\n assert \"\\r\\rlint by rules:\" not in raw_output\n+ assert (\n+ \"DeprecationWarning: The option '--disable_progress_bar' is deprecated, \"\n+ \"use '--disable-progress-bar'\"\n+ ) in raw_output\n \n def test_cli_lint_enabled_progress_bar(\n self, mock_disable_progress_bar: MagicMock\n@@ -1709,7 +1732,7 @@ def test__cli__fix_multiple_errors_no_show_errors():\n args=[\n fix,\n [\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n \"test/fixtures/linter/multiple_sql_errors.sql\",\n ],\n ],\n@@ -1729,7 +1752,7 @@ def test__cli__fix_multiple_errors_show_errors():\n args=[\n fix,\n [\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n \"--show-lint-violations\",\n \"test/fixtures/linter/multiple_sql_errors.sql\",\n ],\n@@ -1771,7 +1794,7 @@ def test__cli__multiple_files__fix_multiple_errors_show_errors():\n args=[\n fix,\n [\n- \"--disable_progress_bar\",\n+ \"--disable-progress-bar\",\n \"--show-lint-violations\",\n sql_path,\n indent_path,\ndiff --git a/test/cli/test_click_deprecated_option.py b/test/cli/test_click_deprecated_option.py\nnew file mode 100644\n--- /dev/null\n+++ b/test/cli/test_click_deprecated_option.py\n@@ -0,0 +1,68 @@\n+\"\"\"The Test suite for `DeprecatedOption` - extension for click options.\"\"\"\n+from typing import List\n+\n+import click\n+import pytest\n+\n+from sqlfluff.cli.click_deprecated_option import (\n+ DeprecatedOption,\n+ DeprecatedOptionsCommand,\n+)\n+from test.cli.commands_test import invoke_assert_code\n+\n+\n+class TestClickDeprecatedOption:\n+ \"\"\"Tests for custom click's option `DeprecatedOption`.\"\"\"\n+\n+ @pytest.mark.parametrize(\n+ \"option, expected_output\",\n+ [\n+ ([], \"{'old_option': False}\\n\"),\n+ (\n+ [\"--old_option\"],\n+ \"DeprecationWarning: The option '--old_option' is deprecated, \"\n+ \"use '--new_option'.\\n{'old_option': True}\\n\",\n+ ),\n+ ([\"--new_option\"], \"{'old_option': True}\\n\"),\n+ ],\n+ )\n+ def test_cli_deprecated_option(\n+ self, option: List[str], expected_output: str\n+ ) -> None:\n+ \"\"\"Prepares command with option which has deprecated version and checks it.\"\"\"\n+\n+ @click.command(cls=DeprecatedOptionsCommand)\n+ @click.option(\n+ \"--old_option\",\n+ \"--new_option\",\n+ is_flag=True,\n+ cls=DeprecatedOption,\n+ deprecated=[\"--old_option\"],\n+ )\n+ def some_command(**kwargs):\n+ click.echo(\"{}\".format(kwargs))\n+\n+ result = invoke_assert_code(args=[some_command, option])\n+ raw_output = result.output\n+\n+ assert raw_output == expected_output\n+\n+ def test_cli_deprecated_option_should_fail_when_missing_attr(\n+ self,\n+ ) -> None:\n+ \"\"\"The DeprecatedOption needs to have specified deprecated attr.\"\"\"\n+\n+ @click.command(cls=DeprecatedOptionsCommand)\n+ @click.option(\n+ \"--old_option\",\n+ \"--new_option\",\n+ is_flag=True,\n+ cls=DeprecatedOption,\n+ )\n+ def some_command(**kwargs):\n+ click.echo(\"{}\".format(kwargs))\n+\n+ with pytest.raises(ValueError) as exc:\n+ invoke_assert_code(args=[some_command, [\"--old_option\"]])\n+\n+ assert str(exc.value) == \"Expected `deprecated` value for `'old_option'`\"\n", "problem_statement": "Standardise `--disable_progress_bar` naming\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nAs noted in https://github.com/sqlfluff/sqlfluff/pull/3610#discussion_r926014745 `--disable_progress_bar` is the only command line option using underscores instead of dashes.\r\n\r\nShould we change this?\r\n\r\nThis would be a breaking change, so do we leave until next major release?\r\nOr do we accept both options?\n\n### Expected Behaviour\n\nWe should be standard in out command line option format\n\n### Observed Behaviour\n\n`--disable_progress_bar` is the only non-standard one\n\n### How to reproduce\n\nN/A\n\n### Dialect\n\nN/A\n\n### Version\n\n1.2.1\n\n### Configuration\n\nN/A\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [X] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "I like the idea (of at least as a transition) or accepting both, but then defaulting to the one consistent with the other options.\nSo this is about changing option `--disable_progress_bar` to `--disable-progress-bar`, right? I think I can take care of that, it was me who introduced it here :)\r\n\r\nAdditionally I would make an attempt to have these two options available, but to nicely inform users that one with underscores is deprecated. What do you think @tunetheweb? \r\n\r\nI see I cannot assign myself to that Issue.\r\n\n> So this is about changing option `--disable_progress_bar` to `--disable-progress-bar`, right? I think I can take care of that, it was me who introduced it here :)\r\n\r\nCorrect and thanks for taking on\r\n\r\n> Additionally I would make an attempt to have these two options available, but to nicely inform users that one with underscores is deprecated. What do you think @tunetheweb?\r\n\r\n@alanmcruickshank added some functionality that might help in #3874 but not sure if that applies to command lines options too (I\u2019m having less time to work on SQLFluff lately so not following it as closely as I used to). If not maybe it should?\r\n\r\n> I see I cannot assign myself to that Issue.\r\n\r\nYeah only maintainers can assign, which is a bit of an annoying restriction of GitHub so we tend not to use that field and commenting (like you\u2019ve done here) is sufficient to claim an issue. Please comment again if to \u201cunassign\u201d yourself if it turns out you won\u2019t be able to work on it after all. Though lack of progress is a usually good indicator of that anyway \ud83d\ude04", "created_at": "2022-10-01T22:18:25Z", "version": "1.3", "FAIL_TO_PASS": ["test/cli/commands_test.py::test__cli__command_directed", "test/cli/commands_test.py::test__cli__command_dialect", "test/cli/commands_test.py::test__cli__command_no_dialect", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_explicit_warning", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_implicit_warning", "test/cli/commands_test.py::test__cli__command_dialect_legacy", "test/cli/commands_test.py::test__cli__command_extra_config_fail", "test/cli/commands_test.py::test__cli__command_lint_stdin[command0]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command1]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command2]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command0]", "test/cli/commands_test.py::test__cli__command_lint_parse[command1]", "test/cli/commands_test.py::test__cli__command_lint_parse[command2]", "test/cli/commands_test.py::test__cli__command_lint_parse[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command4]", "test/cli/commands_test.py::test__cli__command_lint_parse[command5]", "test/cli/commands_test.py::test__cli__command_lint_parse[command6]", "test/cli/commands_test.py::test__cli__command_lint_parse[command7]", "test/cli/commands_test.py::test__cli__command_lint_parse[command8]", "test/cli/commands_test.py::test__cli__command_lint_parse[command9]", "test/cli/commands_test.py::test__cli__command_lint_parse[command10]", "test/cli/commands_test.py::test__cli__command_lint_parse[command11]", "test/cli/commands_test.py::test__cli__command_lint_parse[command12]", "test/cli/commands_test.py::test__cli__command_lint_parse[command13]", "test/cli/commands_test.py::test__cli__command_lint_parse[command14]", "test/cli/commands_test.py::test__cli__command_lint_parse[command15]", "test/cli/commands_test.py::test__cli__command_lint_parse[command16]", "test/cli/commands_test.py::test__cli__command_lint_parse[command17]", "test/cli/commands_test.py::test__cli__command_lint_parse[command18]", "test/cli/commands_test.py::test__cli__command_lint_parse[command19]", "test/cli/commands_test.py::test__cli__command_lint_parse[command20]", "test/cli/commands_test.py::test__cli__command_lint_parse[command21]", "test/cli/commands_test.py::test__cli__command_lint_parse[command22]", "test/cli/commands_test.py::test__cli__command_lint_parse[command23]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command0-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command1-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command2-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command3-1]", "test/cli/commands_test.py::test__cli__command_lint_warning_explicit_file_ignored", "test/cli/commands_test.py::test__cli__command_lint_skip_ignore_files", "test/cli/commands_test.py::test__cli__command_lint_ignore_local_config", "test/cli/commands_test.py::test__cli__command_versioning", "test/cli/commands_test.py::test__cli__command_version", "test/cli/commands_test.py::test__cli__command_rules", "test/cli/commands_test.py::test__cli__command_dialects", "test/cli/commands_test.py::test__cli__command__fix[L001-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/whitespace_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L003-test/fixtures/linter/indentation_error_hard.sql]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_templating_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error_FIX_EVEN_UNPARSABLE]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[2_files_with_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-True]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-True]", "test/cli/commands_test.py::test__cli__fix_loop_limit_behavior[--", "test/cli/commands_test.py::test__cli__command_fix_stdin[select", "test/cli/commands_test.py::test__cli__command_fix_stdin[", "test/cli/commands_test.py::test__cli__command_fix_stdin[SELECT", "test/cli/commands_test.py::test__cli__command_fix_stdin_logging_to_stderr", "test/cli/commands_test.py::test__cli__command_fix_stdin_safety", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[create", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[select", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-y-0-0]", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-n-1-1]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-json]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[select", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[SElect", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command0]", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command1]", "test/cli/commands_test.py::test__cli__command_lint_nocolor", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation_native", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation-native]", "test/cli/commands_test.py::test___main___help", "test/cli/commands_test.py::test_encoding[utf-8-ascii]", "test/cli/commands_test.py::test_encoding[utf-8-sig-UTF-8-SIG]", "test/cli/commands_test.py::test_encoding[utf-32-UTF-32]", "test/cli/commands_test.py::test_cli_encoding[utf-8-command-line-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-command-line-True]", "test/cli/commands_test.py::test_cli_encoding[utf-8-config-file-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-config-file-True]", "test/cli/commands_test.py::test_cli_no_disable_noqa_flag", "test/cli/commands_test.py::test_cli_disable_noqa_flag", "test/cli/commands_test.py::test_cli_get_default_config", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar_deprecated_option", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_paths", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_files", "test/cli/commands_test.py::test__cli__fix_multiple_errors_no_show_errors", "test/cli/commands_test.py::test__cli__fix_multiple_errors_show_errors", "test/cli/commands_test.py::test__cli__multiple_files__fix_multiple_errors_show_errors", "test/cli/test_click_deprecated_option.py::TestClickDeprecatedOption::test_cli_deprecated_option[option0-{'old_option':", "test/cli/test_click_deprecated_option.py::TestClickDeprecatedOption::test_cli_deprecated_option[option1-DeprecationWarning:", "test/cli/test_click_deprecated_option.py::TestClickDeprecatedOption::test_cli_deprecated_option[option2-{'old_option':", "test/cli/test_click_deprecated_option.py::TestClickDeprecatedOption::test_cli_deprecated_option_should_fail_when_missing_attr"], "PASS_TO_PASS": [], "environment_setup_commit": "dc59c2a5672aacedaf91f0e6129b467eefad331b"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-4753", "base_commit": "24178a589c279220c6605324c446122d15ebc3fb", "patch": "diff --git a/docs/generate-rule-docs.py b/docs/generate-rule-docs.py\n--- a/docs/generate-rule-docs.py\n+++ b/docs/generate-rule-docs.py\n@@ -41,8 +41,9 @@\n # Set the bundle name to the ref.\n _bundle_name = f\":ref:`bundle_{bundle}`\"\n for idx, rule in enumerate(rule_bundles[bundle]):\n- aliases = \", \".join(rule.aliases[:3]) + (\n- \",\" if len(rule.aliases) > 3 else \"\"\n+ step = 1 # The number of aliases per line.\n+ aliases = \", \".join(rule.aliases[:step]) + (\n+ \",\" if len(rule.aliases) > step else \"\"\n )\n name_ref = f\":sqlfluff:ref:`{rule.name}`\"\n code_ref = f\":sqlfluff:ref:`{rule.code}`\"\n@@ -51,15 +52,16 @@\n f\"| {code_ref : <28} | {aliases : <18} |\\n\"\n )\n \n- j = 3\n+ j = 1\n+\n while True:\n if not rule.aliases[j:]:\n break\n- aliases = \", \".join(rule.aliases[j : j + 3]) + (\n- \",\" if len(rule.aliases[j:]) > 3 else \"\"\n+ aliases = \", \".join(rule.aliases[j : j + step]) + (\n+ \",\" if len(rule.aliases[j:]) > step else \"\"\n )\n f.write(f\"|{' ' * 42}|{' ' * 50}|{' ' * 30}| {aliases : <18} |\\n\")\n- j += 3\n+ j += step\n \n if idx + 1 < len(rule_bundles[bundle]):\n f.write(f\"|{' ' * 42}+{'-' * 50}+{'-' * 30}+{'-' * 20}+\\n\")\ndiff --git a/src/sqlfluff/rules/layout/LT12.py b/src/sqlfluff/rules/layout/LT12.py\n--- a/src/sqlfluff/rules/layout/LT12.py\n+++ b/src/sqlfluff/rules/layout/LT12.py\n@@ -102,8 +102,10 @@ class Rule_LT12(BaseRule):\n \n \"\"\"\n \n- name = \"layout.end-of-file\"\n- aliases = (\"L009\",)\n+ name = \"layout.end_of_file\"\n+ # Between 2.0.0 and 2.0.4 we supported had a kebab-case name for this rule\n+ # so the old name remains here as an alias to enable backward compatibility.\n+ aliases = (\"L009\", \"layout.end-of-file\")\n groups = (\"all\", \"core\", \"layout\")\n \n targets_templated = True\n", "test_patch": "diff --git a/test/api/simple_test.py b/test/api/simple_test.py\n--- a/test/api/simple_test.py\n+++ b/test/api/simple_test.py\n@@ -95,7 +95,7 @@\n \"line_no\": 1,\n \"line_pos\": 41,\n \"description\": \"Files must end with a single trailing newline.\",\n- \"name\": \"layout.end-of-file\",\n+ \"name\": \"layout.end_of_file\",\n },\n ]\n \n", "problem_statement": "layout.end-of-file is the only rule in kebab case\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nOur rules are all in `snake_case`, except for `layout.end-of-file`\n\n### Expected Behaviour\n\nAll rules should be in snake case\n\n### Observed Behaviour\n\nAs above\n\n### How to reproduce\n\n-\n\n### Dialect\n\nNA\n\n### Version\n\nMain\n\n### Configuration\n\nNA\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [X] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "", "created_at": "2023-04-14T12:49:53Z", "version": "1.4", "FAIL_TO_PASS": ["test/api/simple_test.py::test__api__lint_string"], "PASS_TO_PASS": ["test/api/simple_test.py::test__api__lint_string_without_violations", "test/api/simple_test.py::test__api__lint_string_specific", "test/api/simple_test.py::test__api__lint_string_specific_single", "test/api/simple_test.py::test__api__lint_string_specific_exclude", "test/api/simple_test.py::test__api__lint_string_specific_exclude_single", "test/api/simple_test.py::test__api__lint_string_specific_exclude_all_failed_rules", "test/api/simple_test.py::test__api__fix_string", "test/api/simple_test.py::test__api__fix_string_specific", "test/api/simple_test.py::test__api__fix_string_specific_exclude", "test/api/simple_test.py::test__api__fix_string_unparsable", "test/api/simple_test.py::test__api__fix_string_unparsable_fix_even_unparsable", "test/api/simple_test.py::test__api__parse_string", "test/api/simple_test.py::test__api__parse_fail", "test/api/simple_test.py::test__api__config_path", "test/api/simple_test.py::test__api__config_override[kwargs0-expected0]", "test/api/simple_test.py::test__api__config_override[kwargs1-expected1]", "test/api/simple_test.py::test__api__invalid_dialect"], "environment_setup_commit": "d19de0ecd16d298f9e3bfb91da122734c40c01e5"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-4778", "base_commit": "e3f77d58f56149f9c8db3b790ef263b9853a9cb5", "patch": "diff --git a/src/sqlfluff/core/linter/linter.py b/src/sqlfluff/core/linter/linter.py\n--- a/src/sqlfluff/core/linter/linter.py\n+++ b/src/sqlfluff/core/linter/linter.py\n@@ -189,7 +189,7 @@ def _lex_templated_file(\n getattr(elem, \"indent_val\", 0)\n for elem in cast(Tuple[BaseSegment, ...], tokens)\n )\n- if indent_balance != 0:\n+ if indent_balance != 0: # pragma: no cover\n linter_logger.debug(\n \"Indent balance test failed for %r. Template indents will not be \"\n \"linted for this file.\",\n@@ -207,7 +207,7 @@ def _lex_templated_file(\n if token.indent_val != 0:\n # Don't allow it if we're not linting templating block indents.\n if not templating_blocks_indent:\n- continue\n+ continue # pragma: no cover\n new_tokens.append(token)\n \n # Return new buffer\ndiff --git a/src/sqlfluff/core/templaters/slicers/tracer.py b/src/sqlfluff/core/templaters/slicers/tracer.py\n--- a/src/sqlfluff/core/templaters/slicers/tracer.py\n+++ b/src/sqlfluff/core/templaters/slicers/tracer.py\n@@ -524,7 +524,7 @@ def extract_block_type(tag_name, block_subtype):\n # a block, but its behavior is basically syntactic sugar for\n # {{ open(\"somefile).read() }}. Thus, treat it as templated code.\n # It's a similar situation with {% import %} and {% from ... import %}.\n- if tag_name in [\"include\", \"import\", \"from\"]:\n+ if tag_name in [\"include\", \"import\", \"from\", \"do\"]:\n block_type = \"templated\"\n elif tag_name.startswith(\"end\"):\n block_type = \"block_end\"\n", "test_patch": "diff --git a/test/core/templaters/jinja_test.py b/test/core/templaters/jinja_test.py\n--- a/test/core/templaters/jinja_test.py\n+++ b/test/core/templaters/jinja_test.py\n@@ -1091,6 +1091,23 @@ def _load_result(*args, **kwargs):\n (\"literal\", slice(132, 133, None), slice(34, 35, None)),\n ],\n ),\n+ (\n+ # Tests Jinja \"do\" directive. Should be treated as a\n+ # templated instead of block - issue 4603.\n+ \"\"\"{% do true %}\n+\n+{% if true %}\n+ select 1\n+{% endif %}\"\"\",\n+ None,\n+ [\n+ (\"templated\", slice(0, 13, None), slice(0, 0, None)),\n+ (\"literal\", slice(13, 15, None), slice(0, 2, None)),\n+ (\"block_start\", slice(15, 28, None), slice(2, 2, None)),\n+ (\"literal\", slice(28, 42, None), slice(2, 16, None)),\n+ (\"block_end\", slice(42, 53, None), slice(16, 16, None)),\n+ ],\n+ ),\n (\n # Tests issue 2541, a bug where the {%- endfor %} was causing\n # IndexError: list index out of range.\n", "problem_statement": "2.0.2 - LT02 issues when query contains \"do\" statement.\n### Search before asking\r\n\r\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\r\n\r\n\r\n### What Happened\r\n\r\nSQLFluff v2.0.2 gives LT02 indentation errors for the Jinja `if`-block when `template_blocks_indent` is set to `True`.\r\nThe example SQL below is a bit contrived, but it's the smallest failing example I could produce based on our real SQL.\r\n\r\nIf I remove the Jinja `do`-expression from the code, the `if` block validates without errors.\r\n\r\n### Expected Behaviour\r\n\r\nI expect the SQL to pass the linting tests.\r\n\r\n### Observed Behaviour\r\n\r\nOutput from SQLFluff v2.0.2:\r\n```\r\nL: 5 | P: 1 | LT02 | Line should not be indented.\r\n | [layout.indent]\r\nL: 6 | P: 1 | LT02 | Line should not be indented.\r\n | [layout.indent]\r\n```\r\n\r\n### How to reproduce\r\n\r\nSQL to reproduce:\r\n```\r\n{% set cols = ['a', 'b'] %}\r\n{% do cols.remove('a') %}\r\n\r\n{% if true %}\r\n select a\r\n from some_table\r\n{% endif %}\r\n```\r\n\r\n### Dialect\r\n\r\n`ansi`\r\n\r\n### Version\r\n\r\n```\r\n> sqlfluff --version\r\nsqlfluff, version 2.0.2\r\n\r\n> python --version\r\nPython 3.9.9\r\n```\r\n\r\n### Configuration\r\n\r\n```\r\n[sqlfluff]\r\ndialect = ansi\r\ntemplater = jinja\r\n\r\n[sqlfluff:indentation]\r\ntemplate_blocks_indent = True\r\n```\r\n\r\n### Are you willing to work on and submit a PR to address the issue?\r\n\r\n- [X] Yes I am willing to submit a PR!\r\n\r\n### Code of Conduct\r\n\r\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\r\n\n2.0.2 - LT02 issues when query contains \"do\" statement.\n### Search before asking\r\n\r\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\r\n\r\n\r\n### What Happened\r\n\r\nSQLFluff v2.0.2 gives LT02 indentation errors for the Jinja `if`-block when `template_blocks_indent` is set to `True`.\r\nThe example SQL below is a bit contrived, but it's the smallest failing example I could produce based on our real SQL.\r\n\r\nIf I remove the Jinja `do`-expression from the code, the `if` block validates without errors.\r\n\r\n### Expected Behaviour\r\n\r\nI expect the SQL to pass the linting tests.\r\n\r\n### Observed Behaviour\r\n\r\nOutput from SQLFluff v2.0.2:\r\n```\r\nL: 5 | P: 1 | LT02 | Line should not be indented.\r\n | [layout.indent]\r\nL: 6 | P: 1 | LT02 | Line should not be indented.\r\n | [layout.indent]\r\n```\r\n\r\n### How to reproduce\r\n\r\nSQL to reproduce:\r\n```\r\n{% set cols = ['a', 'b'] %}\r\n{% do cols.remove('a') %}\r\n\r\n{% if true %}\r\n select a\r\n from some_table\r\n{% endif %}\r\n```\r\n\r\n### Dialect\r\n\r\n`ansi`\r\n\r\n### Version\r\n\r\n```\r\n> sqlfluff --version\r\nsqlfluff, version 2.0.2\r\n\r\n> python --version\r\nPython 3.9.9\r\n```\r\n\r\n### Configuration\r\n\r\n```\r\n[sqlfluff]\r\ndialect = ansi\r\ntemplater = jinja\r\n\r\n[sqlfluff:indentation]\r\ntemplate_blocks_indent = True\r\n```\r\n\r\n### Are you willing to work on and submit a PR to address the issue?\r\n\r\n- [X] Yes I am willing to submit a PR!\r\n\r\n### Code of Conduct\r\n\r\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\r\n\n", "hints_text": "I think think this is almost certainly about the `do` statement, hopefully this should be very solvable.\nAny pointers on where I should start looking if I would work on a fix @alanmcruickshank?\n@fredriv - great question. I just had a quick look and this is a very strange bug, but hopefully one with a satisfying solution.\r\n\r\nIf I run `sqlfluff parse` on the file I get this:\r\n\r\n```\r\n[L: 1, P: 1] |file:\r\n[L: 1, P: 1] | [META] placeholder: [Type: 'templated', Raw: \"{% set cols = ['a', 'b'] %}\"]\r\n[L: 1, P: 28] | newline: '\\n'\r\n[L: 2, P: 1] | [META] placeholder: [Type: 'block_start', Raw: \"{% do cols.remove('a') %}\", Block: '230a18']\r\n[L: 2, P: 26] | newline: '\\n'\r\n[L: 3, P: 1] | newline: '\\n'\r\n[L: 4, P: 1] | [META] placeholder: [Type: 'block_start', Raw: '{% if true %}', Block: 'e33036']\r\n[L: 4, P: 14] | newline: '\\n'\r\n[L: 5, P: 1] | whitespace: ' '\r\n[L: 5, P: 5] | statement:\r\n[L: 5, P: 5] | select_statement:\r\n[L: 5, P: 5] | select_clause:\r\n[L: 5, P: 5] | keyword: 'select'\r\n[L: 5, P: 11] | [META] indent:\r\n[L: 5, P: 11] | whitespace: ' '\r\n[L: 5, P: 12] | select_clause_element:\r\n[L: 5, P: 12] | column_reference:\r\n[L: 5, P: 12] | naked_identifier: 'a'\r\n[L: 5, P: 13] | newline: '\\n'\r\n[L: 6, P: 1] | whitespace: ' '\r\n[L: 6, P: 5] | [META] dedent:\r\n[L: 6, P: 5] | from_clause:\r\n[L: 6, P: 5] | keyword: 'from'\r\n[L: 6, P: 9] | whitespace: ' '\r\n[L: 6, P: 10] | from_expression:\r\n[L: 6, P: 10] | [META] indent:\r\n[L: 6, P: 10] | from_expression_element:\r\n[L: 6, P: 10] | table_expression:\r\n[L: 6, P: 10] | table_reference:\r\n[L: 6, P: 10] | naked_identifier: 'some_table'\r\n[L: 6, P: 20] | [META] dedent:\r\n[L: 6, P: 20] | newline: '\\n'\r\n[L: 7, P: 1] | [META] placeholder: [Type: 'block_end', Raw: '{% endif %}', Block: 'e33036']\r\n[L: 7, P: 12] | [META] end_of_file:\r\n```\r\n\r\nNote the difference between that and the output when I remove the `do` line:\r\n\r\n```\r\n[L: 1, P: 1] |file:\r\n[L: 1, P: 1] | [META] placeholder: [Type: 'templated', Raw: \"{% set cols = ['a', 'b'] %}\"]\r\n[L: 1, P: 28] | newline: '\\n'\r\n[L: 2, P: 1] | newline: '\\n'\r\n[L: 3, P: 1] | newline: '\\n'\r\n[L: 4, P: 1] | [META] placeholder: [Type: 'block_start', Raw: '{% if true %}', Block: '0d1e98']\r\n[L: 4, P: 14] | [META] indent: [Block: '0d1e98']\r\n[L: 4, P: 14] | newline: '\\n'\r\n[L: 5, P: 1] | whitespace: ' '\r\n[L: 5, P: 5] | statement:\r\n[L: 5, P: 5] | select_statement:\r\n[L: 5, P: 5] | select_clause:\r\n[L: 5, P: 5] | keyword: 'select'\r\n[L: 5, P: 11] | [META] indent:\r\n[L: 5, P: 11] | whitespace: ' '\r\n[L: 5, P: 12] | select_clause_element:\r\n[L: 5, P: 12] | column_reference:\r\n[L: 5, P: 12] | naked_identifier: 'a'\r\n[L: 5, P: 13] | newline: '\\n'\r\n[L: 6, P: 1] | whitespace: ' '\r\n[L: 6, P: 5] | [META] dedent:\r\n[L: 6, P: 5] | from_clause:\r\n[L: 6, P: 5] | keyword: 'from'\r\n[L: 6, P: 9] | whitespace: ' '\r\n[L: 6, P: 10] | from_expression:\r\n[L: 6, P: 10] | [META] indent:\r\n[L: 6, P: 10] | from_expression_element:\r\n[L: 6, P: 10] | table_expression:\r\n[L: 6, P: 10] | table_reference:\r\n[L: 6, P: 10] | naked_identifier: 'some_table'\r\n[L: 6, P: 20] | [META] dedent:\r\n[L: 6, P: 20] | newline: '\\n'\r\n[L: 7, P: 1] | [META] dedent: [Block: '0d1e98']\r\n[L: 7, P: 1] | [META] placeholder: [Type: 'block_end', Raw: '{% endif %}', Block: '0d1e98']\r\n[L: 7, P: 12] | [META] end_of_file:\r\n```\r\n\r\nSee that in the latter example there are `indent` and `dedent` tokens around the `if` clause, but not in the first example. Something about the `do` call is disrupting the positioning of those indent tokens. Those tokens are inserted during `._iter_segments()` in `lexer.py`, and more specifically in `._handle_zero_length_slice()`. That's probably where you'll find the issue. My guess is that something about the `do` block is throwing off the block tracking?\nThanks! I'll see if I can have a look at it tonight.\r\n\r\nCould it have something to do with the `do` block not having a corresponding `block_end`? \ud83e\udd14\nSo perhaps it should be `templated` instead of `block_start`, similar to the `set` above it?\nIf I add `do` to the list of tag names in `extract_block_type` at https://github.com/sqlfluff/sqlfluff/blob/main/src/sqlfluff/core/templaters/slicers/tracer.py#L527 it regards it as a `templated` element instead of `block_start`, and the indent is added where I want it.\r\n\r\nE.g.\r\n```\r\n[L: 1, P: 1] |file:\r\n[L: 1, P: 1] | [META] placeholder: [Type: 'templated', Raw: \"{% set cols = ['a', 'b'] %}\"]\r\n[L: 1, P: 28] | newline: '\\n'\r\n[L: 2, P: 1] | [META] placeholder: [Type: 'templated', Raw: \"{% do cols.remove('a') %}\"]\r\n[L: 2, P: 26] | newline: '\\n'\r\n[L: 3, P: 1] | newline: '\\n'\r\n[L: 4, P: 1] | [META] placeholder: [Type: 'block_start', Raw: '{% if true %}', Block: '3e39bd']\r\n[L: 4, P: 14] | [META] indent: [Block: '3e39bd']\r\n[L: 4, P: 14] | newline: '\\n'\r\n[L: 5, P: 1] | whitespace: ' '\r\n[L: 5, P: 5] | statement:\r\n[L: 5, P: 5] | select_statement:\r\n[L: 5, P: 5] | select_clause:\r\n[L: 5, P: 5] | keyword: 'select'\r\n[L: 5, P: 11] | [META] indent:\r\n[L: 5, P: 11] | whitespace: ' '\r\n[L: 5, P: 12] | select_clause_element:\r\n[L: 5, P: 12] | column_reference:\r\n[L: 5, P: 12] | naked_identifier: 'a'\r\n[L: 5, P: 13] | newline: '\\n'\r\n[L: 6, P: 1] | whitespace: ' '\r\n[L: 6, P: 5] | [META] dedent:\r\n[L: 6, P: 5] | from_clause:\r\n[L: 6, P: 5] | keyword: 'from'\r\n[L: 6, P: 9] | whitespace: ' '\r\n[L: 6, P: 10] | from_expression:\r\n[L: 6, P: 10] | [META] indent:\r\n[L: 6, P: 10] | from_expression_element:\r\n[L: 6, P: 10] | table_expression:\r\n[L: 6, P: 10] | table_reference:\r\n[L: 6, P: 10] | naked_identifier: 'some_table'\r\n[L: 6, P: 20] | [META] dedent:\r\n[L: 6, P: 20] | newline: '\\n'\r\n[L: 7, P: 1] | [META] dedent: [Block: '3e39bd']\r\n[L: 7, P: 1] | [META] placeholder: [Type: 'block_end', Raw: '{% endif %}', Block: '3e39bd']\r\n[L: 7, P: 12] | newline: '\\n'\r\n[L: 8, P: 1] | [META] end_of_file:\r\n```\nSimilarly if I instead add `do` to the list of trimmed parts in `update_inside_set_call_macro_or_block` at https://github.com/sqlfluff/sqlfluff/blob/main/src/sqlfluff/core/templaters/slicers/tracer.py#L252-L255\r\n\r\nMaybe a better place to put it? What do you think @alanmcruickshank?\n@fredriv - based on the [docs for jinja](https://jinja.palletsprojects.com/en/3.0.x/extensions/#expression-statement) it looks like we should never get a \"do block\" (i.e. `{% do ... %} ... {% enddo %}`, it's only ever just `{% do ... %}`). That means that treating it like a `templated` section is the right route, i.e. we should add it in `extract_block_type` and not in `update_inside_set_call_macro_or_block`.\r\n\r\nThanks for your research - I think this should be a neat solution! \ud83d\ude80 \n\ud83d\udc4d Ok, I can make a PR :)\nI think think this is almost certainly about the `do` statement, hopefully this should be very solvable.\nAny pointers on where I should start looking if I would work on a fix @alanmcruickshank?\n@fredriv - great question. I just had a quick look and this is a very strange bug, but hopefully one with a satisfying solution.\r\n\r\nIf I run `sqlfluff parse` on the file I get this:\r\n\r\n```\r\n[L: 1, P: 1] |file:\r\n[L: 1, P: 1] | [META] placeholder: [Type: 'templated', Raw: \"{% set cols = ['a', 'b'] %}\"]\r\n[L: 1, P: 28] | newline: '\\n'\r\n[L: 2, P: 1] | [META] placeholder: [Type: 'block_start', Raw: \"{% do cols.remove('a') %}\", Block: '230a18']\r\n[L: 2, P: 26] | newline: '\\n'\r\n[L: 3, P: 1] | newline: '\\n'\r\n[L: 4, P: 1] | [META] placeholder: [Type: 'block_start', Raw: '{% if true %}', Block: 'e33036']\r\n[L: 4, P: 14] | newline: '\\n'\r\n[L: 5, P: 1] | whitespace: ' '\r\n[L: 5, P: 5] | statement:\r\n[L: 5, P: 5] | select_statement:\r\n[L: 5, P: 5] | select_clause:\r\n[L: 5, P: 5] | keyword: 'select'\r\n[L: 5, P: 11] | [META] indent:\r\n[L: 5, P: 11] | whitespace: ' '\r\n[L: 5, P: 12] | select_clause_element:\r\n[L: 5, P: 12] | column_reference:\r\n[L: 5, P: 12] | naked_identifier: 'a'\r\n[L: 5, P: 13] | newline: '\\n'\r\n[L: 6, P: 1] | whitespace: ' '\r\n[L: 6, P: 5] | [META] dedent:\r\n[L: 6, P: 5] | from_clause:\r\n[L: 6, P: 5] | keyword: 'from'\r\n[L: 6, P: 9] | whitespace: ' '\r\n[L: 6, P: 10] | from_expression:\r\n[L: 6, P: 10] | [META] indent:\r\n[L: 6, P: 10] | from_expression_element:\r\n[L: 6, P: 10] | table_expression:\r\n[L: 6, P: 10] | table_reference:\r\n[L: 6, P: 10] | naked_identifier: 'some_table'\r\n[L: 6, P: 20] | [META] dedent:\r\n[L: 6, P: 20] | newline: '\\n'\r\n[L: 7, P: 1] | [META] placeholder: [Type: 'block_end', Raw: '{% endif %}', Block: 'e33036']\r\n[L: 7, P: 12] | [META] end_of_file:\r\n```\r\n\r\nNote the difference between that and the output when I remove the `do` line:\r\n\r\n```\r\n[L: 1, P: 1] |file:\r\n[L: 1, P: 1] | [META] placeholder: [Type: 'templated', Raw: \"{% set cols = ['a', 'b'] %}\"]\r\n[L: 1, P: 28] | newline: '\\n'\r\n[L: 2, P: 1] | newline: '\\n'\r\n[L: 3, P: 1] | newline: '\\n'\r\n[L: 4, P: 1] | [META] placeholder: [Type: 'block_start', Raw: '{% if true %}', Block: '0d1e98']\r\n[L: 4, P: 14] | [META] indent: [Block: '0d1e98']\r\n[L: 4, P: 14] | newline: '\\n'\r\n[L: 5, P: 1] | whitespace: ' '\r\n[L: 5, P: 5] | statement:\r\n[L: 5, P: 5] | select_statement:\r\n[L: 5, P: 5] | select_clause:\r\n[L: 5, P: 5] | keyword: 'select'\r\n[L: 5, P: 11] | [META] indent:\r\n[L: 5, P: 11] | whitespace: ' '\r\n[L: 5, P: 12] | select_clause_element:\r\n[L: 5, P: 12] | column_reference:\r\n[L: 5, P: 12] | naked_identifier: 'a'\r\n[L: 5, P: 13] | newline: '\\n'\r\n[L: 6, P: 1] | whitespace: ' '\r\n[L: 6, P: 5] | [META] dedent:\r\n[L: 6, P: 5] | from_clause:\r\n[L: 6, P: 5] | keyword: 'from'\r\n[L: 6, P: 9] | whitespace: ' '\r\n[L: 6, P: 10] | from_expression:\r\n[L: 6, P: 10] | [META] indent:\r\n[L: 6, P: 10] | from_expression_element:\r\n[L: 6, P: 10] | table_expression:\r\n[L: 6, P: 10] | table_reference:\r\n[L: 6, P: 10] | naked_identifier: 'some_table'\r\n[L: 6, P: 20] | [META] dedent:\r\n[L: 6, P: 20] | newline: '\\n'\r\n[L: 7, P: 1] | [META] dedent: [Block: '0d1e98']\r\n[L: 7, P: 1] | [META] placeholder: [Type: 'block_end', Raw: '{% endif %}', Block: '0d1e98']\r\n[L: 7, P: 12] | [META] end_of_file:\r\n```\r\n\r\nSee that in the latter example there are `indent` and `dedent` tokens around the `if` clause, but not in the first example. Something about the `do` call is disrupting the positioning of those indent tokens. Those tokens are inserted during `._iter_segments()` in `lexer.py`, and more specifically in `._handle_zero_length_slice()`. That's probably where you'll find the issue. My guess is that something about the `do` block is throwing off the block tracking?\nThanks! I'll see if I can have a look at it tonight.\r\n\r\nCould it have something to do with the `do` block not having a corresponding `block_end`? \ud83e\udd14\nSo perhaps it should be `templated` instead of `block_start`, similar to the `set` above it?\nIf I add `do` to the list of tag names in `extract_block_type` at https://github.com/sqlfluff/sqlfluff/blob/main/src/sqlfluff/core/templaters/slicers/tracer.py#L527 it regards it as a `templated` element instead of `block_start`, and the indent is added where I want it.\r\n\r\nE.g.\r\n```\r\n[L: 1, P: 1] |file:\r\n[L: 1, P: 1] | [META] placeholder: [Type: 'templated', Raw: \"{% set cols = ['a', 'b'] %}\"]\r\n[L: 1, P: 28] | newline: '\\n'\r\n[L: 2, P: 1] | [META] placeholder: [Type: 'templated', Raw: \"{% do cols.remove('a') %}\"]\r\n[L: 2, P: 26] | newline: '\\n'\r\n[L: 3, P: 1] | newline: '\\n'\r\n[L: 4, P: 1] | [META] placeholder: [Type: 'block_start', Raw: '{% if true %}', Block: '3e39bd']\r\n[L: 4, P: 14] | [META] indent: [Block: '3e39bd']\r\n[L: 4, P: 14] | newline: '\\n'\r\n[L: 5, P: 1] | whitespace: ' '\r\n[L: 5, P: 5] | statement:\r\n[L: 5, P: 5] | select_statement:\r\n[L: 5, P: 5] | select_clause:\r\n[L: 5, P: 5] | keyword: 'select'\r\n[L: 5, P: 11] | [META] indent:\r\n[L: 5, P: 11] | whitespace: ' '\r\n[L: 5, P: 12] | select_clause_element:\r\n[L: 5, P: 12] | column_reference:\r\n[L: 5, P: 12] | naked_identifier: 'a'\r\n[L: 5, P: 13] | newline: '\\n'\r\n[L: 6, P: 1] | whitespace: ' '\r\n[L: 6, P: 5] | [META] dedent:\r\n[L: 6, P: 5] | from_clause:\r\n[L: 6, P: 5] | keyword: 'from'\r\n[L: 6, P: 9] | whitespace: ' '\r\n[L: 6, P: 10] | from_expression:\r\n[L: 6, P: 10] | [META] indent:\r\n[L: 6, P: 10] | from_expression_element:\r\n[L: 6, P: 10] | table_expression:\r\n[L: 6, P: 10] | table_reference:\r\n[L: 6, P: 10] | naked_identifier: 'some_table'\r\n[L: 6, P: 20] | [META] dedent:\r\n[L: 6, P: 20] | newline: '\\n'\r\n[L: 7, P: 1] | [META] dedent: [Block: '3e39bd']\r\n[L: 7, P: 1] | [META] placeholder: [Type: 'block_end', Raw: '{% endif %}', Block: '3e39bd']\r\n[L: 7, P: 12] | newline: '\\n'\r\n[L: 8, P: 1] | [META] end_of_file:\r\n```\nSimilarly if I instead add `do` to the list of trimmed parts in `update_inside_set_call_macro_or_block` at https://github.com/sqlfluff/sqlfluff/blob/main/src/sqlfluff/core/templaters/slicers/tracer.py#L252-L255\r\n\r\nMaybe a better place to put it? What do you think @alanmcruickshank?\n@fredriv - based on the [docs for jinja](https://jinja.palletsprojects.com/en/3.0.x/extensions/#expression-statement) it looks like we should never get a \"do block\" (i.e. `{% do ... %} ... {% enddo %}`, it's only ever just `{% do ... %}`). That means that treating it like a `templated` section is the right route, i.e. we should add it in `extract_block_type` and not in `update_inside_set_call_macro_or_block`.\r\n\r\nThanks for your research - I think this should be a neat solution! \ud83d\ude80 \n\ud83d\udc4d Ok, I can make a PR :)", "created_at": "2023-04-18T04:35:29Z", "version": "1.4", "FAIL_TO_PASS": ["test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%"], "PASS_TO_PASS": ["test/core/templaters/jinja_test.py::test__templater_jinja[simple]", "test/core/templaters/jinja_test.py::test__templater_jinja[unboundlocal_bugfix]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_left_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_and_templated_whitespace]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_block_hard]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_right_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_comment]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[union_all_loop1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[set_multiple_variables_and_define_macro]", "test/core/templaters/jinja_test.py::test_templater_set_block_handling", "test/core/templaters/jinja_test.py::test__templater_jinja_error_variable", "test/core/templaters/jinja_test.py::test__templater_jinja_dynamic_variable_no_violations", "test/core/templaters/jinja_test.py::test__templater_jinja_error_syntax", "test/core/templaters/jinja_test.py::test__templater_jinja_error_catastrophic", "test/core/templaters/jinja_test.py::test__templater_jinja_error_macro_path_does_not_exist", "test/core/templaters/jinja_test.py::test__templater_jinja_lint_empty", "test/core/templaters/jinja_test.py::test__templater_full[jinja_a/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_b/jinja-False-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_config-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_is_incremental-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_ref-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_source-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_this-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_var_default-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins_test-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_e/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_f/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_g_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_h_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag_2-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_j_libraries/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_k_config_override_path_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/001-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/002-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/003-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/004-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/005-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/006-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/007-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/008-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/009-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_m_libraries_module/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_n_nested_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_o_config_override_dbt_builtins/override_dbt_builtins-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_p_disable_dbt_builtins/disable_dbt_builtins-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_q_multiple_path_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_jinja_block_matching", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[{%", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[-None-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[foo-None-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{{", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT\\n", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%-", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[select\\n", "test/core/templaters/jinja_test.py::test__templater_jinja_large_file_check", "test/core/templaters/jinja_test.py::test_jinja_undefined_callable[-expected_violation0]", "test/core/templaters/jinja_test.py::test_jinja_undefined_callable[templating-None]", "test/core/templaters/jinja_test.py::test_dummy_undefined_fail_with_undefined_error", "test/core/templaters/jinja_test.py::test_undefined_magic_methods"], "environment_setup_commit": "d19de0ecd16d298f9e3bfb91da122734c40c01e5"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-2907", "base_commit": "305159ea643baf6b4744b98c3566613754b2f659", "patch": "diff --git a/src/sqlfluff/core/templaters/jinja.py b/src/sqlfluff/core/templaters/jinja.py\n--- a/src/sqlfluff/core/templaters/jinja.py\n+++ b/src/sqlfluff/core/templaters/jinja.py\n@@ -343,20 +343,33 @@ def process(\n # first Exception which serves only to catch catastrophic errors.\n try:\n syntax_tree = env.parse(in_str)\n- undefined_variables = meta.find_undeclared_variables(syntax_tree)\n+ potentially_undefined_variables = meta.find_undeclared_variables(\n+ syntax_tree\n+ )\n except Exception as err: # pragma: no cover\n # TODO: Add a url here so people can get more help.\n raise SQLTemplaterError(f\"Failure in identifying Jinja variables: {err}.\")\n \n- # Get rid of any that *are* actually defined.\n- for val in live_context:\n- if val in undefined_variables:\n- undefined_variables.remove(val)\n+ undefined_variables = set()\n+\n+ class Undefined:\n+ \"\"\"Similar to jinja2.StrictUndefined, but remembers, not fails.\"\"\"\n+\n+ def __init__(self, name):\n+ self.name = name\n+\n+ def __str__(self):\n+ \"\"\"Treat undefined vars as empty, but remember for later.\"\"\"\n+ undefined_variables.add(self.name)\n+ return \"\"\n+\n+ def __getattr__(self, item):\n+ undefined_variables.add(self.name)\n+ return Undefined(f\"{self.name}.{item}\")\n \n- if undefined_variables:\n- # Lets go through and find out where they are:\n- for val in self._crawl_tree(syntax_tree, undefined_variables, in_str):\n- violations.append(val)\n+ for val in potentially_undefined_variables:\n+ if val not in live_context:\n+ live_context[val] = Undefined(name=val)\n \n try:\n # NB: Passing no context. Everything is loaded when the template is loaded.\n@@ -368,6 +381,10 @@ def process(\n config=config,\n make_template=make_template,\n )\n+ if undefined_variables:\n+ # Lets go through and find out where they are:\n+ for val in self._crawl_tree(syntax_tree, undefined_variables, in_str):\n+ violations.append(val)\n return (\n TemplatedFile(\n source_str=in_str,\n", "test_patch": "diff --git a/test/core/templaters/jinja_test.py b/test/core/templaters/jinja_test.py\n--- a/test/core/templaters/jinja_test.py\n+++ b/test/core/templaters/jinja_test.py\n@@ -411,6 +411,20 @@ def test__templater_jinja_error_variable():\n assert any(v.rule_code() == \"TMP\" and v.line_no == 1 for v in vs)\n \n \n+def test__templater_jinja_dynamic_variable_no_violations():\n+ \"\"\"Test no templater violation for variable defined within template.\"\"\"\n+ t = JinjaTemplater(override_context=dict(blah=\"foo\"))\n+ instr = \"\"\"{% if True %}\n+ {% set some_var %}1{% endset %}\n+ SELECT {{some_var}}\n+{% endif %}\n+\"\"\"\n+ outstr, vs = t.process(in_str=instr, fname=\"test\", config=FluffConfig())\n+ assert str(outstr) == \"\\n \\n SELECT 1\\n\\n\"\n+ # Check we have no violations.\n+ assert len(vs) == 0\n+\n+\n def test__templater_jinja_error_syntax():\n \"\"\"Test syntax problems in the jinja templater.\"\"\"\n t = JinjaTemplater()\n", "problem_statement": "sqlfluff doesn't recognise a jinja variable set inside of \"if\" statement\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nWhen I try to define a jinja variable using \"set\" jinja directive inside of an \"if\" jinja statement, sqlfluff complains: \r\n\"Undefined jinja template variable\".\n\n### Expected Behaviour\n\nto not have a linting issue\n\n### Observed Behaviour\n\nsqlfluff lint gives an error:\r\n\"Undefined jinja template variable\"\n\n### How to reproduce\n\ntry to create a \"temp.sql\" file with the following content\r\n\r\n```\r\n{% if True %}\r\n {% set some_var %}1{% endset %}\r\n SELECT {{some_var}}\r\n{% endif %}\r\n```\r\n\r\nand run:\r\n```\r\nsqlfluff lint ./temp.sql\r\n```\r\n\r\nYou will get the following error:\r\n```\r\n== [./temp.sql] FAIL \r\nL: 2 | P: 12 | TMP | Undefined jinja template variable: 'some_var'\r\nL: 3 | P: 14 | TMP | Undefined jinja template variable: 'some_var'\r\n```\n\n### Dialect\n\ntested on 'snowflake' dialect\n\n### Version\n\nsqlfluff, version 0.11.1\r\nPython 3.8.12\n\n### Configuration\n\n[sqlfluff]\r\nverbose = 1\r\ndialect = snowflake\r\ntemplater = jinja\r\nexclude_rules = L027,L031,L032,L036,L044,L046,L034,L050\r\noutput_line_length = 121\r\nsql_file_exts=.sql\r\n\r\n[sqlfluff:rules]\r\ntab_space_size = 4\r\nmax_line_length = 250\r\nindent_unit = space\r\ncomma_style = trailing\r\nallow_scalar = True\r\nsingle_table_references = consistent\r\nunquoted_identifiers_policy = aliases\r\n\r\n[sqlfluff:rules:L042]\r\nforbid_subquery_in = both\r\n\r\n[sqlfluff:rules:L010] # Keywords\r\ncapitalisation_policy = upper\r\n\r\n[sqlfluff:rules:L014]\r\nextended_capitalisation_policy = lower\r\n\r\n[sqlfluff:rules:L030] # function names\r\nextended_capitalisation_policy = upper\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [ ] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "Does Jinja support this? I don't see how this could be a SQLFluff issue.\nYour example does work on this website. I wonder if there's a Jinja runtime setting that affects whether this works.\r\n\r\nhttp://jinja.quantprogramming.com/\nIt also works with `j2cli` on my local machine. Seems like this _has_ to be a Jinja runtime setting...\r\n\r\nhttps://github.com/kolypto/j2cli\nIt was added in Jinja 2.8: https://jinja.palletsprojects.com/en/3.0.x/templates/#block-assignments\r\n\r\nNot sure what version we pull in depending on our other dependencies?\r\n\nI'm digging into this more. SQLFluff contains some additional code that attempts to detect undeclared Jinja variables and provide better error handling. The \"issue\" is being detected and reported by that code, not by Jinja itself. So we should be able to fix this. Need to do this carefully so we don't break error reporting for real errors. \nI think I have a fix. Just need to make the undefined variable check more sophisticated.", "created_at": "2022-03-23T21:32:13Z", "version": "0.10", "FAIL_TO_PASS": ["test/core/templaters/jinja_test.py::test__templater_jinja_dynamic_variable_no_violations"], "PASS_TO_PASS": ["test/core/templaters/jinja_test.py::test__templater_jinja[simple]", "test/core/templaters/jinja_test.py::test__templater_jinja[unboundlocal_bugfix]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_left_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_right_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_comment]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[union_all_loop1]", "test/core/templaters/jinja_test.py::test_templater_set_block_handling", "test/core/templaters/jinja_test.py::test__templater_jinja_error_variable", "test/core/templaters/jinja_test.py::test__templater_jinja_error_syntax", "test/core/templaters/jinja_test.py::test__templater_jinja_error_catastrophic", "test/core/templaters/jinja_test.py::test__templater_jinja_error_macro_path_does_not_exist", "test/core/templaters/jinja_test.py::test__templater_jinja_lint_empty", "test/core/templaters/jinja_test.py::test__templater_full[jinja_a/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_b/jinja-False-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/var_default-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_e/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_f/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_g_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_h_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag_2-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_j_libraries/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_k_config_override_path_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/001-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/002-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_m_libraries_module/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_n_nested_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[{%", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[-None-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[foo-None-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{{", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT\\n", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%-", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[select\\n"], "environment_setup_commit": "3d52e8270d82aeccf4c516d059a80a6947919aea"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-4051", "base_commit": "c3defb095b1aa7fe23c4bd430fdff2ce6ed6161d", "patch": "diff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py\n--- a/src/sqlfluff/cli/commands.py\n+++ b/src/sqlfluff/cli/commands.py\n@@ -137,24 +137,14 @@ def red_log_filter(record: logging.LogRecord) -> bool:\n class PathAndUserErrorHandler:\n \"\"\"Make an API call but with error handling for the CLI.\"\"\"\n \n- def __init__(self, formatter, paths):\n+ def __init__(self, formatter):\n self.formatter = formatter\n- self.paths = paths\n \n def __enter__(self):\n return self\n \n def __exit__(self, exc_type, exc_val, exc_tb):\n- if exc_type is OSError:\n- click.echo(\n- self.formatter.colorize(\n- f\"The path(s) { self.paths } could not be \"\n- \"accessed. Check it/they exist(s).\",\n- Color.red,\n- )\n- )\n- sys.exit(EXIT_ERROR)\n- elif exc_type is SQLFluffUserError:\n+ if exc_type is SQLFluffUserError:\n click.echo(\n \"\\nUser Error: \"\n + self.formatter.colorize(\n@@ -584,7 +574,7 @@ def lint(\n if verbose >= 1:\n click.echo(format_linting_result_header())\n \n- with PathAndUserErrorHandler(formatter, paths):\n+ with PathAndUserErrorHandler(formatter):\n # add stdin if specified via lone '-'\n if (\"-\",) == paths:\n result = lnt.lint_string_wrapped(sys.stdin.read(), fname=\"stdin\")\n@@ -833,7 +823,7 @@ def fix(\n # Lint the paths (not with the fix argument at this stage), outputting as we go.\n click.echo(\"==== finding fixable violations ====\")\n \n- with PathAndUserErrorHandler(formatter, paths):\n+ with PathAndUserErrorHandler(formatter):\n result = lnt.lint_paths(\n paths,\n fix=True,\n@@ -1051,7 +1041,7 @@ def parse(\n t0 = time.monotonic()\n \n # handle stdin if specified via lone '-'\n- with PathAndUserErrorHandler(formatter, path):\n+ with PathAndUserErrorHandler(formatter):\n if \"-\" == path:\n parsed_strings = [\n lnt.parse_string(\n@@ -1156,7 +1146,7 @@ def render(\n )\n \n # handle stdin if specified via lone '-'\n- with PathAndUserErrorHandler(formatter, path):\n+ with PathAndUserErrorHandler(formatter):\n if \"-\" == path:\n raw_sql = sys.stdin.read()\n fname = \"stdin\"\ndiff --git a/src/sqlfluff/core/linter/linter.py b/src/sqlfluff/core/linter/linter.py\n--- a/src/sqlfluff/core/linter/linter.py\n+++ b/src/sqlfluff/core/linter/linter.py\n@@ -27,6 +27,7 @@\n SQLLintError,\n SQLParseError,\n SQLFluffSkipFile,\n+ SQLFluffUserError,\n )\n from sqlfluff.core.parser import Lexer, Parser, RegexLexer\n from sqlfluff.core.file_helpers import get_encoding\n@@ -969,7 +970,9 @@ def paths_from_path(\n if ignore_non_existent_files:\n return []\n else:\n- raise OSError(\"Specified path does not exist\")\n+ raise SQLFluffUserError(\n+ f\"Specified path does not exist. Check it/they exist(s): {path}.\"\n+ )\n \n # Files referred to exactly are also ignored if\n # matched, but we warn the users when that happens\n", "test_patch": "diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py\n--- a/test/cli/commands_test.py\n+++ b/test/cli/commands_test.py\n@@ -1203,7 +1203,10 @@ def test__cli__command_lint_serialize_from_stdin(serialize, sql, expected, exit_\n def test__cli__command_fail_nice_not_found(command):\n \"\"\"Check commands fail as expected when then don't find files.\"\"\"\n result = invoke_assert_code(args=command, ret_code=2)\n- assert \"could not be accessed\" in result.output\n+ assert (\n+ \"User Error: Specified path does not exist. Check it/they \"\n+ \"exist(s): this_file_does_not_exist.sql\"\n+ ) in result.output\n \n \n @patch(\"click.utils.should_strip_ansi\")\ndiff --git a/test/core/linter_test.py b/test/core/linter_test.py\n--- a/test/core/linter_test.py\n+++ b/test/core/linter_test.py\n@@ -16,6 +16,7 @@\n SQLBaseError,\n SQLLintError,\n SQLParseError,\n+ SQLFluffUserError,\n )\n from sqlfluff.cli.formatters import OutputStreamFormatter\n from sqlfluff.cli.outputstream import make_output_stream\n@@ -120,9 +121,9 @@ def test__linter__skip_large_bytes(filesize, raises_skip):\n \n \n def test__linter__path_from_paths__not_exist():\n- \"\"\"Test extracting paths from a file path.\"\"\"\n+ \"\"\"Test that the right errors are raise when a file doesn't exist.\"\"\"\n lntr = Linter()\n- with pytest.raises(IOError):\n+ with pytest.raises(SQLFluffUserError):\n lntr.paths_from_path(\"asflekjfhsakuefhse\")\n \n \n", "problem_statement": "Misleading path does not exist message\nIt looks like if _at least one_ of the paths provided to sqlfluff do not exist, it will display an error message implying that _all_ of the supplied paths do not exist:\r\n\r\n```bash\r\ndbt@b54bee9ced88:/workspaces/dbt-dutchie$ sqlfluff fix models/shared/dispensaries.sql models/shares/dispensary_chains.sql\r\n==== finding fixable violations ====\r\n=== [dbt templater] Compiling dbt project...\r\n== [models/shared/dispensaries.sql] FAIL\r\nL: 6 | P: 2 | L003 | Indentation not consistent with line #376\r\nL: 8 | P: 2 | L003 | Indentation not consistent with line #376\r\nL: 9 | P: 3 | L003 | Line over-indented compared to line #376\r\nL: 10 | P: 2 | L003 | Indentation not consistent with line #376\r\nL: 12 | P: 2 | L003 | Indentation not consistent with line #376\r\nL: 13 | P: 3 | L003 | Line over-indented compared to line #376\r\nL: 14 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 15 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 16 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 17 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 18 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 19 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 20 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 21 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 22 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 23 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 24 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 25 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 26 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 27 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 28 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 29 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 30 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 31 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 32 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 33 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 34 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 58 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 35 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 36 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 37 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 38 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 39 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 40 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 41 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 42 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 43 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 44 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 45 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 46 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 47 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 48 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 49 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 50 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 51 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 52 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 53 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 54 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 55 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 56 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 57 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 58 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 59 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 60 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 61 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 62 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 63 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 64 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 65 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 66 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 67 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 68 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 69 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 70 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 71 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 72 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 73 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 74 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 75 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 76 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 77 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 78 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 79 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 80 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 81 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 82 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 83 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 84 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 85 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 86 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 87 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 88 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 89 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 90 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 91 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 92 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 92 | P: 44 | L001 | Unnecessary trailing whitespace.\r\nL: 93 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 94 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 95 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 96 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 97 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 98 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 99 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 100 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 101 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 102 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 103 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 104 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 105 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 106 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 107 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 108 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 109 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 110 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 111 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 112 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 113 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 114 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 115 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 116 | P: 3 | L003 | Line over-indented compared to line #376\r\nL: 235 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 117 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 118 | P: 3 | L003 | Line over-indented compared to line #376\r\nL: 119 | P: 4 | L003 | Line over-indented compared to line #376\r\nL: 120 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 121 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 122 | P: 2 | L003 | Indentation not consistent with line #376\r\nL: 339 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 343 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 347 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 351 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 355 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 358 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 361 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 364 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 367 | P: 1 | L004 | Incorrect indentation type found in file.\r\nL: 370 | P: 1 | L004 | Incorrect indentation type found in file.\r\nThe path(s) ('models/shared/dispensaries.sql', 'models/shares/dispensary_chains.sql') could not be accessed. Check it/they exist(s).\r\n```\r\n\r\n## Expected Behaviour\r\nI would expect only the unaccessible paths to be included in the error message.\r\n\r\n## Observed Behaviour\r\nSee above\r\n\r\n## Version\r\n```bash\r\ndbt@b54bee9ced88:/workspaces/dbt-dutchie$ sqlfluff --version\r\nsqlfluff, version 0.5.2\r\n```\r\n\r\n```bash\r\ndbt@b54bee9ced88:/workspaces/dbt-dutchie$ python --version\r\nPython 3.8.6\r\n```\r\n\r\n## Configuration\r\n```\r\n[sqlfluff]\r\ndialect = snowflake\r\ntemplater = dbt\r\nrules = L001,L002,L003,L004,L005,L006\r\nignore = parsing,templating\r\n\r\n[sqlfluff:rules]\r\nmax_line_length = 120\r\ncomma_style = trailing\r\n\r\n[sqlfluff:rules:L010]\r\ncapitalisation_policy = upper\r\n```\r\n\n", "hints_text": "", "created_at": "2022-11-09T18:11:06Z", "version": "1.3", "FAIL_TO_PASS": ["test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command0]", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command1]", "test/core/linter_test.py::test__linter__path_from_paths__not_exist"], "PASS_TO_PASS": ["test/cli/commands_test.py::test__cli__command_directed", "test/cli/commands_test.py::test__cli__command_dialect", "test/cli/commands_test.py::test__cli__command_no_dialect", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_explicit_warning", "test/cli/commands_test.py::test__cli__command_parse_error_dialect_implicit_warning", "test/cli/commands_test.py::test__cli__command_dialect_legacy", "test/cli/commands_test.py::test__cli__command_extra_config_fail", "test/cli/commands_test.py::test__cli__command_lint_stdin[command0]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command1]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command2]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command3]", "test/cli/commands_test.py::test__cli__command_render_stdin", "test/cli/commands_test.py::test__cli__command_lint_parse[command0]", "test/cli/commands_test.py::test__cli__command_lint_parse[command1]", "test/cli/commands_test.py::test__cli__command_lint_parse[command2]", "test/cli/commands_test.py::test__cli__command_lint_parse[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command4]", "test/cli/commands_test.py::test__cli__command_lint_parse[command5]", "test/cli/commands_test.py::test__cli__command_lint_parse[command6]", "test/cli/commands_test.py::test__cli__command_lint_parse[command7]", "test/cli/commands_test.py::test__cli__command_lint_parse[command8]", "test/cli/commands_test.py::test__cli__command_lint_parse[command9]", "test/cli/commands_test.py::test__cli__command_lint_parse[command10]", "test/cli/commands_test.py::test__cli__command_lint_parse[command11]", "test/cli/commands_test.py::test__cli__command_lint_parse[command12]", "test/cli/commands_test.py::test__cli__command_lint_parse[command13]", "test/cli/commands_test.py::test__cli__command_lint_parse[command14]", "test/cli/commands_test.py::test__cli__command_lint_parse[command15]", "test/cli/commands_test.py::test__cli__command_lint_parse[command16]", "test/cli/commands_test.py::test__cli__command_lint_parse[command17]", "test/cli/commands_test.py::test__cli__command_lint_parse[command18]", "test/cli/commands_test.py::test__cli__command_lint_parse[command19]", "test/cli/commands_test.py::test__cli__command_lint_parse[command20]", "test/cli/commands_test.py::test__cli__command_lint_parse[command21]", "test/cli/commands_test.py::test__cli__command_lint_parse[command22]", "test/cli/commands_test.py::test__cli__command_lint_parse[command23]", "test/cli/commands_test.py::test__cli__command_lint_parse[command24]", "test/cli/commands_test.py::test__cli__command_lint_parse[command25]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command0-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command1-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command2-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command3-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command4-1]", "test/cli/commands_test.py::test__cli__command_lint_warning_explicit_file_ignored", "test/cli/commands_test.py::test__cli__command_lint_skip_ignore_files", "test/cli/commands_test.py::test__cli__command_lint_ignore_local_config", "test/cli/commands_test.py::test__cli__command_versioning", "test/cli/commands_test.py::test__cli__command_version", "test/cli/commands_test.py::test__cli__command_rules", "test/cli/commands_test.py::test__cli__command_dialects", "test/cli/commands_test.py::test__cli__command__fix[L001-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/whitespace_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L003-test/fixtures/linter/indentation_error_hard.sql]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_templating_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error_FIX_EVEN_UNPARSABLE]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[2_files_with_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-True]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-True]", "test/cli/commands_test.py::test__cli__fix_loop_limit_behavior[--", "test/cli/commands_test.py::test__cli__command_fix_stdin[select", "test/cli/commands_test.py::test__cli__command_fix_stdin[", "test/cli/commands_test.py::test__cli__command_fix_stdin[SELECT", "test/cli/commands_test.py::test__cli__command_fix_stdin_logging_to_stderr", "test/cli/commands_test.py::test__cli__command_fix_stdin_safety", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[create", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[select", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-y-0-0]", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-n-1-1]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-json]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[select", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[SElect", "test/cli/commands_test.py::test__cli__command_lint_nocolor", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation_native", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation-native]", "test/cli/commands_test.py::test___main___help", "test/cli/commands_test.py::test_encoding[utf-8-ascii]", "test/cli/commands_test.py::test_encoding[utf-8-sig-UTF-8-SIG]", "test/cli/commands_test.py::test_encoding[utf-32-UTF-32]", "test/cli/commands_test.py::test_cli_encoding[utf-8-command-line-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-command-line-True]", "test/cli/commands_test.py::test_cli_encoding[utf-8-config-file-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-config-file-True]", "test/cli/commands_test.py::test_cli_no_disable_noqa_flag", "test/cli/commands_test.py::test_cli_disable_noqa_flag", "test/cli/commands_test.py::test_cli_get_default_config", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar_deprecated_option", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_paths", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_files", "test/cli/commands_test.py::test__cli__fix_multiple_errors_no_show_errors", "test/cli/commands_test.py::test__cli__fix_multiple_errors_show_errors", "test/cli/commands_test.py::test__cli__multiple_files__fix_multiple_errors_show_errors", "test/cli/commands_test.py::test__cli__render_fail", "test/cli/commands_test.py::test__cli__render_pass", "test/core/linter_test.py::test__linter__path_from_paths__dir", "test/core/linter_test.py::test__linter__path_from_paths__default", "test/core/linter_test.py::test__linter__path_from_paths__exts", "test/core/linter_test.py::test__linter__path_from_paths__file", "test/core/linter_test.py::test__linter__skip_large_bytes[0-False]", "test/core/linter_test.py::test__linter__skip_large_bytes[5-True]", "test/core/linter_test.py::test__linter__skip_large_bytes[2000-False]", "test/core/linter_test.py::test__linter__path_from_paths__not_exist_ignore", "test/core/linter_test.py::test__linter__path_from_paths__explicit_ignore", "test/core/linter_test.py::test__linter__path_from_paths__sqlfluffignore_current_directory", "test/core/linter_test.py::test__linter__path_from_paths__dot", "test/core/linter_test.py::test__linter__path_from_paths__ignore[test/fixtures/linter/sqlfluffignore]", "test/core/linter_test.py::test__linter__path_from_paths__ignore[test/fixtures/linter/sqlfluffignore/]", "test/core/linter_test.py::test__linter__path_from_paths__ignore[test/fixtures/linter/sqlfluffignore/.]", "test/core/linter_test.py::test__linter__lint_string_vs_file[test/fixtures/linter/indentation_errors.sql]", "test/core/linter_test.py::test__linter__lint_string_vs_file[test/fixtures/linter/whitespace_errors.sql]", "test/core/linter_test.py::test__linter__get_violations_filter_rules[None-7]", "test/core/linter_test.py::test__linter__get_violations_filter_rules[L010-2]", "test/core/linter_test.py::test__linter__get_violations_filter_rules[rules2-2]", "test/core/linter_test.py::test__linter__linting_result__sum_dicts", "test/core/linter_test.py::test__linter__linting_result__combine_dicts", "test/core/linter_test.py::test__linter__linting_result_check_tuples_by_path[False-list]", "test/core/linter_test.py::test__linter__linting_result_check_tuples_by_path[True-dict]", "test/core/linter_test.py::test__linter__linting_result_get_violations[1]", "test/core/linter_test.py::test__linter__linting_result_get_violations[2]", "test/core/linter_test.py::test__linter__linting_parallel_thread[False]", "test/core/linter_test.py::test__linter__linting_parallel_thread[True]", "test/core/linter_test.py::test_lint_path_parallel_wrapper_exception", "test/core/linter_test.py::test__linter__get_runner_processes[512-1-1]", "test/core/linter_test.py::test__linter__get_runner_processes[512-0-512]", "test/core/linter_test.py::test__linter__get_runner_processes[512--12-500]", "test/core/linter_test.py::test__linter__get_runner_processes[512-5-5]", "test/core/linter_test.py::test__linter__get_runner_processes[1--1-1]", "test/core/linter_test.py::test__linter__linting_unexpected_error_handled_gracefully", "test/core/linter_test.py::test__linter__raises_malformed_noqa", "test/core/linter_test.py::test__linter__empty_file", "test/core/linter_test.py::test__linter__mask_templated_violations[True-check_tuples0]", "test/core/linter_test.py::test__linter__mask_templated_violations[False-check_tuples1]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8.sql-autodetect-False]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8-sig.sql-autodetect-False]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8.sql-utf-8-False]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8-sig.sql-utf-8-True]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8.sql-utf-8-sig-False]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8-sig.sql-utf-8-sig-False]", "test/core/linter_test.py::test_parse_noqa[-None]", "test/core/linter_test.py::test_parse_noqa[noqa-expected1]", "test/core/linter_test.py::test_parse_noqa[noqa?-SQLParseError]", "test/core/linter_test.py::test_parse_noqa[noqa:-expected3]", "test/core/linter_test.py::test_parse_noqa[noqa:L001,L002-expected4]", "test/core/linter_test.py::test_parse_noqa[noqa:", "test/core/linter_test.py::test_parse_noqa[Inline", "test/core/linter_test.py::test_parse_noqa_no_dups", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_no_ignore]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_specific_line]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_different_specific_line]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_different_specific_rule]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_enable_this_range]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_disable_this_range]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_1_ignore_disable_specific_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_2_ignore_disable_specific_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_3_ignore_disable_specific_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_4_ignore_disable_specific_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_1_ignore_disable_all_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_2_ignore_disable_all_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_3_ignore_disable_all_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_4_ignore_disable_all_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[4_violations_two_types_disable_specific_enable_all]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[4_violations_two_types_disable_all_enable_specific]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violations_comment_inline_ignore]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[2_violations_comment_inline_ignore]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violations_comment_inline_glob_ignore]", "test/core/linter_test.py::test_linter_noqa", "test/core/linter_test.py::test_linter_noqa_with_templating", "test/core/linter_test.py::test_linter_noqa_template_errors", "test/core/linter_test.py::test_linter_noqa_prs", "test/core/linter_test.py::test_linter_noqa_tmp", "test/core/linter_test.py::test_linter_noqa_disable", "test/core/linter_test.py::test_delayed_exception", "test/core/linter_test.py::test__attempt_to_change_templater_warning", "test/core/linter_test.py::test_safe_create_replace_file[utf8_create]", "test/core/linter_test.py::test_safe_create_replace_file[utf8_update]", "test/core/linter_test.py::test_safe_create_replace_file[utf8_special_char]", "test/core/linter_test.py::test_safe_create_replace_file[incorrect_encoding]", "test/core/linter_test.py::test_advanced_api_methods", "test/core/linter_test.py::test_normalise_newlines", "test/core/linter_test.py::test_require_match_parse_grammar"], "environment_setup_commit": "dc59c2a5672aacedaf91f0e6129b467eefad331b"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-905", "base_commit": "62e8dc3a148c40c0c28f62b23e943692a3198846", "patch": "diff --git a/src/sqlfluff/core/rules/std/L036.py b/src/sqlfluff/core/rules/std/L036.py\n--- a/src/sqlfluff/core/rules/std/L036.py\n+++ b/src/sqlfluff/core/rules/std/L036.py\n@@ -90,21 +90,26 @@ def _get_indexes(segment):\n )\n \n def _eval_multiple_select_target_elements(self, select_targets_info, segment):\n- if select_targets_info.first_new_line_idx == -1:\n- # there are multiple select targets but no new lines\n-\n- # Find and delete any whitespace between \"SELECT\" and its targets.\n- ws_to_delete = segment.select_children(\n- start_seg=segment.segments[select_targets_info.select_idx],\n- select_if=lambda s: s.is_type(\"whitespace\"),\n- loop_while=lambda s: s.is_type(\"whitespace\") or s.is_meta,\n+ \"\"\"Multiple select targets. Ensure each is on a separate line.\"\"\"\n+ # Insert newline before every select target.\n+ fixes = []\n+ for i, select_target in enumerate(select_targets_info.select_targets):\n+ base_segment = (\n+ segment if not i else select_targets_info.select_targets[i - 1]\n )\n- fixes = [LintFix(\"delete\", ws) for ws in ws_to_delete]\n- # Insert newline before the first select target.\n- ins = self.make_newline(\n- pos_marker=segment.pos_marker.advance_by(segment.raw)\n- )\n- fixes.append(LintFix(\"create\", select_targets_info.select_targets[0], ins))\n+ if base_segment.pos_marker.line_no == select_target.pos_marker.line_no:\n+ # Find and delete any whitespace before the select target.\n+ ws_to_delete = segment.select_children(\n+ start_seg=segment.segments[select_targets_info.select_idx]\n+ if not i\n+ else select_targets_info.select_targets[i - 1],\n+ select_if=lambda s: s.is_type(\"whitespace\"),\n+ loop_while=lambda s: s.is_type(\"whitespace\", \"comma\") or s.is_meta,\n+ )\n+ fixes += [LintFix(\"delete\", ws) for ws in ws_to_delete]\n+ ins = self.make_newline(pos_marker=select_target.pos_marker)\n+ fixes.append(LintFix(\"create\", select_target, ins))\n+ if fixes:\n return LintResult(anchor=segment, fixes=fixes)\n \n def _eval_single_select_target_element(\n", "test_patch": "diff --git a/test/api/simple_test.py b/test/api/simple_test.py\n--- a/test/api/simple_test.py\n+++ b/test/api/simple_test.py\n@@ -121,7 +121,14 @@ def test__api__fix_string():\n # Check return types.\n assert isinstance(result, str)\n # Check actual result\n- assert result == \"SELECT\\n *, 1, blah AS foo FROM mytable\\n\"\n+ assert (\n+ result\n+ == \"\"\"SELECT\n+ *,\n+ 1,\n+ blah AS foo FROM mytable\n+\"\"\"\n+ )\n \n \n def test__api__fix_string_specific():\ndiff --git a/test/fixtures/rules/std_rule_cases/L036.yml b/test/fixtures/rules/std_rule_cases/L036.yml\n--- a/test/fixtures/rules/std_rule_cases/L036.yml\n+++ b/test/fixtures/rules/std_rule_cases/L036.yml\n@@ -37,7 +37,7 @@ test_multiple_select_targets_all_on_the_same_line:\n fail_str: |\n select a, b, c\n from x\n- fix_str: \"select\\na, b, c\\nfrom x\\n\"\n+ fix_str: \"select\\na,\\nb,\\nc\\nfrom x\\n\"\n \n test_multiple_select_targets_trailing_whitespace_after_select:\n # TRICKY: Use explicit newlines to preserve the trailing space after \"SELECT\".\n@@ -57,3 +57,47 @@ test_comment_between_select_and_single_select_target:\n -- This is the user's ID.\n FROM\n safe_user\n+\n+test_multiple_select_targets_some_newlines_missing_1:\n+ fail_str: |\n+ select\n+ a, b, c,\n+ d, e, f, g,\n+ h\n+ from x\n+ # The spaces before a, d, and h look odd, but these are places where the\n+ # select targets were already on a separate line, and the rule made no\n+ # changes.\n+ fix_str: |\n+ select\n+ a,\n+ b,\n+ c,\n+ d,\n+ e,\n+ f,\n+ g,\n+ h\n+ from x\n+\n+\n+test_multiple_select_targets_some_newlines_missing_2:\n+ fail_str: |\n+ select a, b, c,\n+ d, e, f, g,\n+ h\n+ from x\n+ # The spaces before d, and h look odd, but these are places where the\n+ # select targets were already on a separate line, and the rule made no\n+ # changes.\n+ fix_str: |\n+ select\n+ a,\n+ b,\n+ c,\n+ d,\n+ e,\n+ f,\n+ g,\n+ h\n+ from x\n", "problem_statement": "Enhance rule L036 to put all columns on separate lines if any of them are\nThe current description is ambiguous, but after discussion, we decided to update the rule and keep the description at least _similar_ to what it is currently.. See discussion on #769.\n", "hints_text": "FWIW I'm a +1 for this...", "created_at": "2021-03-28T21:22:12Z", "version": "0.4", "FAIL_TO_PASS": ["test/api/simple_test.py::test__api__fix_string"], "PASS_TO_PASS": ["test/api/simple_test.py::test__api__lint_string_without_violations", "test/api/simple_test.py::test__api__lint_string", "test/api/simple_test.py::test__api__lint_file", "test/api/simple_test.py::test__api__lint_string_specific", "test/api/simple_test.py::test__api__fix_string_specific", "test/api/simple_test.py::test__api__parse_string"], "environment_setup_commit": "cbdcfb09feb4883de91de142956c3be6ac7f827d"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-1625", "base_commit": "14e1a23a3166b9a645a16de96f694c77a5d4abb7", "patch": "diff --git a/src/sqlfluff/rules/L031.py b/src/sqlfluff/rules/L031.py\n--- a/src/sqlfluff/rules/L031.py\n+++ b/src/sqlfluff/rules/L031.py\n@@ -211,7 +211,7 @@ def _lint_aliases_in_join(\n violation_buff.append(\n LintResult(\n anchor=alias_info.alias_identifier_ref,\n- description=\"Avoid using aliases in join condition\",\n+ description=\"Avoid aliases in from clauses and join conditions.\",\n fixes=fixes,\n )\n )\n", "test_patch": "diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py\n--- a/test/cli/commands_test.py\n+++ b/test/cli/commands_test.py\n@@ -49,7 +49,7 @@ def invoke_assert_code(\n expected_output = \"\"\"== [test/fixtures/linter/indentation_error_simple.sql] FAIL\n L: 2 | P: 4 | L003 | Indentation not hanging or a multiple of 4 spaces\n L: 5 | P: 10 | L010 | Keywords must be consistently upper case.\n-L: 5 | P: 13 | L031 | Avoid using aliases in join condition\n+L: 5 | P: 13 | L031 | Avoid aliases in from clauses and join conditions.\n \"\"\"\n \n \n", "problem_statement": "TSQL - L031 incorrectly triggers \"Avoid using aliases in join condition\" when no join present\n## Expected Behaviour\r\n\r\nBoth of these queries should pass, the only difference is the addition of a table alias 'a':\r\n\r\n1/ no alias\r\n\r\n```\r\nSELECT [hello]\r\nFROM\r\n mytable\r\n```\r\n\r\n2/ same query with alias\r\n\r\n```\r\nSELECT a.[hello]\r\nFROM\r\n mytable AS a\r\n```\r\n\r\n## Observed Behaviour\r\n\r\n1/ passes\r\n2/ fails with: L031: Avoid using aliases in join condition.\r\n\r\nBut there is no join condition :-)\r\n\r\n## Steps to Reproduce\r\n\r\nLint queries above\r\n\r\n## Dialect\r\n\r\nTSQL\r\n\r\n## Version\r\n\r\nsqlfluff 0.6.9\r\nPython 3.6.9\r\n\r\n## Configuration\r\n\r\nN/A\n", "hints_text": "Actually, re-reading the docs I think this is the intended behaviour... closing", "created_at": "2021-10-13T11:35:29Z", "version": "0.6", "FAIL_TO_PASS": ["test/cli/commands_test.py::test__cli__command_directed"], "PASS_TO_PASS": ["test/cli/commands_test.py::test__cli__command_dialect", "test/cli/commands_test.py::test__cli__command_dialect_legacy", "test/cli/commands_test.py::test__cli__command_lint_stdin[command0]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command1]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command2]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command0]", "test/cli/commands_test.py::test__cli__command_lint_parse[command1]", "test/cli/commands_test.py::test__cli__command_lint_parse[command2]", "test/cli/commands_test.py::test__cli__command_lint_parse[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command4]", "test/cli/commands_test.py::test__cli__command_lint_parse[command5]", "test/cli/commands_test.py::test__cli__command_lint_parse[command6]", "test/cli/commands_test.py::test__cli__command_lint_parse[command7]", "test/cli/commands_test.py::test__cli__command_lint_parse[command8]", "test/cli/commands_test.py::test__cli__command_lint_parse[command9]", "test/cli/commands_test.py::test__cli__command_lint_parse[command10]", "test/cli/commands_test.py::test__cli__command_lint_parse[command11]", "test/cli/commands_test.py::test__cli__command_lint_parse[command12]", "test/cli/commands_test.py::test__cli__command_lint_parse[command13]", "test/cli/commands_test.py::test__cli__command_lint_parse[command14]", "test/cli/commands_test.py::test__cli__command_lint_parse[command15]", "test/cli/commands_test.py::test__cli__command_lint_parse[command16]", "test/cli/commands_test.py::test__cli__command_lint_parse[command17]", "test/cli/commands_test.py::test__cli__command_lint_parse[command18]", "test/cli/commands_test.py::test__cli__command_lint_parse[command19]", "test/cli/commands_test.py::test__cli__command_lint_parse[command20]", "test/cli/commands_test.py::test__cli__command_lint_parse[command21]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command0-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command1-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command2-1]", "test/cli/commands_test.py::test__cli__command_lint_warning_explicit_file_ignored", "test/cli/commands_test.py::test__cli__command_lint_skip_ignore_files", "test/cli/commands_test.py::test__cli__command_versioning", "test/cli/commands_test.py::test__cli__command_version", "test/cli/commands_test.py::test__cli__command_rules", "test/cli/commands_test.py::test__cli__command_dialects", "test/cli/commands_test.py::test__cli__command__fix[L001-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/whitespace_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L003-test/fixtures/linter/indentation_error_hard.sql]", "test/cli/commands_test.py::test__cli__command_fix_stdin[select", "test/cli/commands_test.py::test__cli__command_fix_stdin[", "test/cli/commands_test.py::test__cli__command_fix_stdin[SELECT", "test/cli/commands_test.py::test__cli__command_fix_stdin_logging_to_stderr", "test/cli/commands_test.py::test__cli__command_fix_stdin_safety", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[create", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[select", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-y-0-0]", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-n-65-1]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[select", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[SElect", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command0]", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command1]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation", "test/cli/commands_test.py::test___main___help", "test/cli/commands_test.py::test_encoding[utf-8-ascii]", "test/cli/commands_test.py::test_encoding[utf-8-sig-UTF-8-SIG]", "test/cli/commands_test.py::test_encoding[utf-32-UTF-32]"], "environment_setup_commit": "67023b85c41d23d6c6d69812a41b207c4f8a9331"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-2326", "base_commit": "38cff664d9505999fb7473a4a7b29ba36aba7883", "patch": "diff --git a/src/sqlfluff/core/parser/__init__.py b/src/sqlfluff/core/parser/__init__.py\n--- a/src/sqlfluff/core/parser/__init__.py\n+++ b/src/sqlfluff/core/parser/__init__.py\n@@ -23,6 +23,7 @@\n Delimited,\n Bracketed,\n AnyNumberOf,\n+ AnySetOf,\n Ref,\n Anything,\n Nothing,\n@@ -56,6 +57,7 @@\n \"Delimited\",\n \"Bracketed\",\n \"AnyNumberOf\",\n+ \"AnySetOf\",\n \"Ref\",\n \"Anything\",\n \"Nothing\",\ndiff --git a/src/sqlfluff/core/parser/grammar/__init__.py b/src/sqlfluff/core/parser/grammar/__init__.py\n--- a/src/sqlfluff/core/parser/grammar/__init__.py\n+++ b/src/sqlfluff/core/parser/grammar/__init__.py\n@@ -1,7 +1,12 @@\n \"\"\"Definitions of grammars.\"\"\"\n \n from sqlfluff.core.parser.grammar.base import Ref, Anything, Nothing\n-from sqlfluff.core.parser.grammar.anyof import AnyNumberOf, OneOf, OptionallyBracketed\n+from sqlfluff.core.parser.grammar.anyof import (\n+ AnyNumberOf,\n+ AnySetOf,\n+ OneOf,\n+ OptionallyBracketed,\n+)\n from sqlfluff.core.parser.grammar.delimited import Delimited\n from sqlfluff.core.parser.grammar.greedy import GreedyUntil, StartsWith\n from sqlfluff.core.parser.grammar.sequence import Sequence, Bracketed\n@@ -12,6 +17,7 @@\n \"Anything\",\n \"Nothing\",\n \"AnyNumberOf\",\n+ \"AnySetOf\",\n \"OneOf\",\n \"OptionallyBracketed\",\n \"Delimited\",\ndiff --git a/src/sqlfluff/core/parser/grammar/anyof.py b/src/sqlfluff/core/parser/grammar/anyof.py\n--- a/src/sqlfluff/core/parser/grammar/anyof.py\n+++ b/src/sqlfluff/core/parser/grammar/anyof.py\n@@ -22,6 +22,7 @@ class AnyNumberOf(BaseGrammar):\n def __init__(self, *args, **kwargs):\n self.max_times = kwargs.pop(\"max_times\", None)\n self.min_times = kwargs.pop(\"min_times\", 0)\n+ self.max_times_per_element = kwargs.pop(\"max_times_per_element\", None)\n # Any patterns to _prevent_ a match.\n self.exclude = kwargs.pop(\"exclude\", None)\n super().__init__(*args, **kwargs)\n@@ -121,7 +122,7 @@ def _prune_options(\n \n def _match_once(\n self, segments: Tuple[BaseSegment, ...], parse_context: ParseContext\n- ) -> MatchResult:\n+ ) -> Tuple[MatchResult, Optional[\"MatchableType\"]]:\n \"\"\"Match the forward segments against the available elements once.\n \n This serves as the main body of OneOf, but also a building block\n@@ -141,14 +142,14 @@ def _match_once(\n return MatchResult.from_unmatched(segments)\n \n with parse_context.deeper_match() as ctx:\n- match, _ = self._longest_trimmed_match(\n+ match, matched_option = self._longest_trimmed_match(\n segments,\n available_options,\n parse_context=ctx,\n trim_noncode=False,\n )\n \n- return match\n+ return match, matched_option\n \n @match_wrapper()\n @allow_ephemeral\n@@ -171,6 +172,13 @@ def match(\n matched_segments: MatchResult = MatchResult.from_empty()\n unmatched_segments: Tuple[BaseSegment, ...] = segments\n n_matches = 0\n+\n+ # Keep track of the number of times each option has been matched.\n+ available_options, _ = self._prune_options(\n+ segments, parse_context=parse_context\n+ )\n+ available_option_counter = {str(o): 0 for o in available_options}\n+\n while True:\n if self.max_times and n_matches >= self.max_times:\n # We've matched as many times as we can\n@@ -197,7 +205,23 @@ def match(\n else:\n pre_seg = () # empty tuple\n \n- match = self._match_once(unmatched_segments, parse_context=parse_context)\n+ match, matched_option = self._match_once(\n+ unmatched_segments, parse_context=parse_context\n+ )\n+\n+ # Increment counter for matched option.\n+ if matched_option and (str(matched_option) in available_option_counter):\n+ available_option_counter[str(matched_option)] += 1\n+ # Check if we have matched an option too many times.\n+ if (\n+ self.max_times_per_element\n+ and available_option_counter[str(matched_option)]\n+ > self.max_times_per_element\n+ ):\n+ return MatchResult(\n+ matched_segments.matched_segments, unmatched_segments\n+ )\n+\n if match:\n matched_segments += pre_seg + match.matched_segments\n unmatched_segments = match.unmatched_segments\n@@ -240,3 +264,10 @@ def __init__(self, *args, **kwargs):\n args[0] if len(args) == 1 else Sequence(*args),\n **kwargs,\n )\n+\n+\n+class AnySetOf(AnyNumberOf):\n+ \"\"\"Match any number of the elements but each element can only be matched once.\"\"\"\n+\n+ def __init__(self, *args, **kwargs):\n+ super().__init__(*args, max_times_per_element=1, **kwargs)\ndiff --git a/src/sqlfluff/dialects/dialect_ansi.py b/src/sqlfluff/dialects/dialect_ansi.py\n--- a/src/sqlfluff/dialects/dialect_ansi.py\n+++ b/src/sqlfluff/dialects/dialect_ansi.py\n@@ -19,6 +19,7 @@\n from sqlfluff.core.dialects.common import AliasInfo, ColumnAliasInfo\n from sqlfluff.core.parser import (\n AnyNumberOf,\n+ AnySetOf,\n Anything,\n BaseFileSegment,\n BaseSegment,\n@@ -515,7 +516,7 @@\n ),\n optional=True,\n ),\n- AnyNumberOf(\n+ AnySetOf(\n # ON DELETE clause, e.g. ON DELETE NO ACTION\n Sequence(\n \"ON\",\n", "test_patch": "diff --git a/test/core/parser/grammar_test.py b/test/core/parser/grammar_test.py\n--- a/test/core/parser/grammar_test.py\n+++ b/test/core/parser/grammar_test.py\n@@ -12,6 +12,7 @@\n Indent,\n )\n from sqlfluff.core.parser.context import RootParseContext\n+from sqlfluff.core.parser.grammar.anyof import AnySetOf\n from sqlfluff.core.parser.segments import EphemeralSegment, BaseSegment\n from sqlfluff.core.parser.grammar.base import BaseGrammar\n from sqlfluff.core.parser.grammar.noncode import NonCodeMatcher\n@@ -678,3 +679,22 @@ def test__parser__grammar_noncode(seg_list, fresh_ansi_dialect):\n m = NonCodeMatcher().match(seg_list[1:], parse_context=ctx)\n # We should match one and only one segment\n assert len(m) == 1\n+\n+\n+def test__parser__grammar_anysetof(generate_test_segments):\n+ \"\"\"Test the AnySetOf grammar.\"\"\"\n+ token_list = [\"bar\", \" \\t \", \"foo\", \" \\t \", \"bar\"]\n+ seg_list = generate_test_segments(token_list)\n+\n+ bs = StringParser(\"bar\", KeywordSegment)\n+ fs = StringParser(\"foo\", KeywordSegment)\n+ g = AnySetOf(fs, bs)\n+ with RootParseContext(dialect=None) as ctx:\n+ # Check directly\n+ assert g.match(seg_list, parse_context=ctx).matched_segments == (\n+ KeywordSegment(\"bar\", seg_list[0].pos_marker),\n+ WhitespaceSegment(\" \\t \", seg_list[1].pos_marker),\n+ KeywordSegment(\"foo\", seg_list[2].pos_marker),\n+ )\n+ # Check with a bit of whitespace\n+ assert not g.match(seg_list[1:], parse_context=ctx)\n", "problem_statement": "`AnySetOf` grammar\n\r\nI know this has been talked about before in PRs so making an issue to formally track.\r\n\r\nIn many grammars there's a common situation where we have to denote several options that can be specified in any order but they cannot be specified more than once.\r\n\r\nOur general approach to this in the project has been denote this using `AnyNumberOf` as this allows for the different orderings:\r\n```python\r\nAnyNumberOf(\r\n ,\r\n ,\r\n ...\r\n)\r\n```\r\nHowever, the issue with this is that it places no limit on how many times each option can be specified.\r\n\r\nThis means that sqlfluff allows certain invalid statements to parse e.g.\r\n```sql\r\nCREATE TABLE ktw_account_binding (\r\n ktw_id VARCHAR(32) NOT NULL REFERENCES ref_table(bla)\r\n ON DELETE RESTRICT ON DELETE CASCADE ON DELETE CASCADE ON DELETE CASCADE\r\n);\r\n```\r\nhttps://github.com/sqlfluff/sqlfluff/pull/2315#issuecomment-1013847846\r\n\r\nWe've accepted this limitation for the time being as it's more important to get the statements parsing for linting/formatting purposes rather than exactly reflecting the grammar (we'd expect a general degree of common sense when it comes to repeating these options).\r\n\r\nThat being said it would be nice to address this to refine our grammar and reduce dealing with contributor confusion.\r\n\r\n`AnySetOf` would essentially allow all of it's grammar arguments to be parsed in any order a maximum of 1 time each. Hopefully we can inherit from `AnyNumberOf` to simplify this.\n", "hints_text": "", "created_at": "2022-01-16T22:52:45Z", "version": "0.8", "FAIL_TO_PASS": ["test/core/parser/grammar_test.py::test__parser__grammar__base__longest_trimmed_match__basic[seg_list_slice0-matcher_keywords0-False-result_slice0]", "test/core/parser/grammar_test.py::test__parser__grammar__base__longest_trimmed_match__basic[seg_list_slice1-matcher_keywords1-True-result_slice1]", "test/core/parser/grammar_test.py::test__parser__grammar__base__longest_trimmed_match__basic[seg_list_slice2-matcher_keywords2-False-None]", "test/core/parser/grammar_test.py::test__parser__grammar__base__longest_trimmed_match__basic[seg_list_slice3-matcher_keywords3-True-result_slice3]", "test/core/parser/grammar_test.py::test__parser__grammar__base__longest_trimmed_match__adv", "test/core/parser/grammar_test.py::test__parser__grammar__base__look_ahead_match[seg_list_slice0-matcher_keywords0-result_slice0-bar-None]", "test/core/parser/grammar_test.py::test__parser__grammar__base__look_ahead_match[seg_list_slice1-matcher_keywords1-result_slice1-foo-pre_match_slice1]", "test/core/parser/grammar_test.py::test__parser__grammar__base__ephemeral_segment", "test/core/parser/grammar_test.py::test__parser__grammar__oneof__ephemeral_segment", "test/core/parser/grammar_test.py::test__parser__grammar__base__bracket_sensitive_look_ahead_match", "test/core/parser/grammar_test.py::test__parser__grammar__base__bracket_fail_with_open_paren_close_square_mismatch", "test/core/parser/grammar_test.py::test__parser__grammar__base__bracket_fail_with_unexpected_end_bracket", "test/core/parser/grammar_test.py::test__parser__grammar__ref_eq", "test/core/parser/grammar_test.py::test__parser__grammar__oneof__copy", "test/core/parser/grammar_test.py::test__parser__grammar_oneof[True]", "test/core/parser/grammar_test.py::test__parser__grammar_oneof[False]", "test/core/parser/grammar_test.py::test__parser__grammar_oneof_templated", "test/core/parser/grammar_test.py::test__parser__grammar_oneof_exclude", "test/core/parser/grammar_test.py::test__parser__grammar_oneof_take_longest_match", "test/core/parser/grammar_test.py::test__parser__grammar_oneof_take_first", "test/core/parser/grammar_test.py::test__parser__grammar_startswith_a[baar-False]", "test/core/parser/grammar_test.py::test__parser__grammar_startswith_a[bar-True]", "test/core/parser/grammar_test.py::test__parser__grammar_startswith_b[False-3]", "test/core/parser/grammar_test.py::test__parser__grammar_startswith_b[True-4]", "test/core/parser/grammar_test.py::test__parser__grammar_sequence", "test/core/parser/grammar_test.py::test__parser__grammar_sequence_nested", "test/core/parser/grammar_test.py::test__parser__grammar_sequence_indent", "test/core/parser/grammar_test.py::test__parser__grammar_sequence_indent_conditional", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list0-None-True-False-5]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list1-None-True-False-6]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list2-None-True-False-0]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list3-None-True-True-3]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list4-0-True-False-5]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list5-0-False-False-1]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list6-1-True-False-5]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list7-1-False-False-0]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list8-None-True-False-3]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list9-None-False-False-3]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list10-1-True-False-3]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list11-1-False-False-3]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list12-1-False-False-3]", "test/core/parser/grammar_test.py::test__parser__grammar_delimited[token_list13-2-True-False-0]", "test/core/parser/grammar_test.py::test__parser__grammar_greedyuntil[foo-False-1]", "test/core/parser/grammar_test.py::test__parser__grammar_greedyuntil[bar-False-0]", "test/core/parser/grammar_test.py::test__parser__grammar_greedyuntil[baar-False-3]", "test/core/parser/grammar_test.py::test__parser__grammar_greedyuntil[baar-True-6]", "test/core/parser/grammar_test.py::test__parser__grammar_greedyuntil_bracketed", "test/core/parser/grammar_test.py::test__parser__grammar_anything", "test/core/parser/grammar_test.py::test__parser__grammar_nothing", "test/core/parser/grammar_test.py::test__parser__grammar_noncode", "test/core/parser/grammar_test.py::test__parser__grammar_anysetof"], "PASS_TO_PASS": [], "environment_setup_commit": "a5c4eae4e3e419fe95460c9afd9cf39a35a470c4"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-3220", "base_commit": "262010b91cf5616de242dad504c788e9cd33ac58", "patch": "diff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py\n--- a/src/sqlfluff/cli/commands.py\n+++ b/src/sqlfluff/cli/commands.py\n@@ -674,6 +674,7 @@ def do_fixes(lnt, result, formatter=None, **kwargs):\n @click.option(\n \"--FIX-EVEN-UNPARSABLE\",\n is_flag=True,\n+ default=None,\n help=(\n \"Enables fixing of files that have templating or parse errors. \"\n \"Note that the similar-sounding '--ignore' or 'noqa' features merely \"\n@@ -750,7 +751,7 @@ def fix(\n )\n click.echo(\n colorize(\n- \"Use --fix-even-unparsable' to attempt to fix the SQL anyway.\",\n+ \"Use --FIX-EVEN-UNPARSABLE' to attempt to fix the SQL anyway.\",\n Color.red,\n ),\n err=True,\n", "test_patch": "diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py\n--- a/test/cli/commands_test.py\n+++ b/test/cli/commands_test.py\n@@ -687,6 +687,70 @@ def test__cli__fix_error_handling_behavior(sql, fix_args, fixed, exit_code, tmpd\n assert not fixed_path.is_file()\n \n \n+@pytest.mark.parametrize(\n+ \"method,fix_even_unparsable\",\n+ [\n+ (\"command-line\", False),\n+ (\"command-line\", True),\n+ (\"config-file\", False),\n+ (\"config-file\", True),\n+ ],\n+)\n+def test_cli_fix_even_unparsable(\n+ method: str, fix_even_unparsable: bool, monkeypatch, tmpdir\n+):\n+ \"\"\"Test the fix_even_unparsable option works from cmd line and config.\"\"\"\n+ sql_filename = \"fix_even_unparsable.sql\"\n+ sql_path = str(tmpdir / sql_filename)\n+ with open(sql_path, \"w\") as f:\n+ print(\n+ \"\"\"SELECT my_col\n+FROM my_schema.my_table\n+where processdate ! 3\n+\"\"\",\n+ file=f,\n+ )\n+ options = [\n+ \"--dialect\",\n+ \"ansi\",\n+ \"-f\",\n+ \"--fixed-suffix=FIXED\",\n+ sql_path,\n+ ]\n+ if method == \"command-line\":\n+ if fix_even_unparsable:\n+ options.append(\"--FIX-EVEN-UNPARSABLE\")\n+ else:\n+ assert method == \"config-file\"\n+ with open(str(tmpdir / \".sqlfluff\"), \"w\") as f:\n+ print(f\"[sqlfluff]\\nfix_even_unparsable = {fix_even_unparsable}\", file=f)\n+ # TRICKY: Switch current directory to the one with the SQL file. Otherwise,\n+ # the setting doesn't work. That's because SQLFluff reads it in\n+ # sqlfluff.cli.commands.fix(), prior to reading any file-specific settings\n+ # (down in sqlfluff.core.linter.Linter._load_raw_file_and_config()).\n+ monkeypatch.chdir(str(tmpdir))\n+ invoke_assert_code(\n+ ret_code=0 if fix_even_unparsable else 1,\n+ args=[\n+ fix,\n+ options,\n+ ],\n+ )\n+ fixed_path = str(tmpdir / \"fix_even_unparsableFIXED.sql\")\n+ if fix_even_unparsable:\n+ with open(fixed_path, \"r\") as f:\n+ fixed_sql = f.read()\n+ assert (\n+ fixed_sql\n+ == \"\"\"SELECT my_col\n+FROM my_schema.my_table\n+WHERE processdate ! 3\n+\"\"\"\n+ )\n+ else:\n+ assert not os.path.isfile(fixed_path)\n+\n+\n _old_eval = BaseRule._eval\n _fix_counter = 0\n \n", "problem_statement": "Config for fix_even_unparsable not being applied\n### Search before asking\r\n\r\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\r\n\r\n\r\n### What Happened\r\n\r\nWhen setting the any config file to `fix_even_unparsable = True` the config get's overriden by the default (or lack thereof) on the @click.option decorator for the fix command.\r\n\r\n### Expected Behaviour\r\n\r\nWhen setting the config `fix_even_unparsable` it should be captured by the fix command as well.\r\n\r\n### Observed Behaviour\r\n\r\nThe `fix_even_unparsable` command is not being captured by the fix command\r\n\r\n### How to reproduce\r\n\r\nCreate a config file and include `fix_even_unparsable`\r\nRun `sqlfluff fix`\r\nNote that `fix_even_unparsable` is set to False at runtime\r\n\r\n### Dialect\r\n\r\nAny\r\n\r\n### Version\r\n\r\n0.13.0\r\n\r\n### Configuration\r\n\r\n`pyproject.toml`\r\n\r\n```\r\n[tool.sqlfluff.core]\r\nverbose = 2\r\ndialect = \"snowflake\"\r\nfix_even_unparsable = true\r\n```\r\n\r\n### Are you willing to work on and submit a PR to address the issue?\r\n\r\n- [X] Yes I am willing to submit a PR!\r\n\r\n### Code of Conduct\r\n\r\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\r\n\n", "hints_text": "I believe the fix would be to just add a `default=None,` to the @click.option decorator.\r\nThis is simple enough for me to create a PR but I don't know how to create tests (or if just adding it is enough) for it as required on the PR template.\n> I believe the fix would be to just add a `default=None,` to the @click.option decorator.\r\nConfirmed that worked\r\n\r\n> This is simple enough for me to create a PR but I don't know how to create tests (or if just adding it is enough) for it as required on the PR template.\r\n\r\nIt would be good to have a test. If you look at `test/fixtures/linter/autofix/snowflake/001_semi_structured` you can see a similar test that uses a .sqlfluff config file for the test run.\nI'm happy to take this unless you want to do it, @pekapa. I fixed a very similar issue with the `--encoding` option a few weeks ago.", "created_at": "2022-04-28T19:13:54Z", "version": "0.12", "FAIL_TO_PASS": ["test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-True]"], "PASS_TO_PASS": ["test/cli/commands_test.py::test__cli__command_directed", "test/cli/commands_test.py::test__cli__command_dialect", "test/cli/commands_test.py::test__cli__command_dialect_legacy", "test/cli/commands_test.py::test__cli__command_extra_config_fail", "test/cli/commands_test.py::test__cli__command_lint_stdin[command0]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command1]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command2]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command0]", "test/cli/commands_test.py::test__cli__command_lint_parse[command1]", "test/cli/commands_test.py::test__cli__command_lint_parse[command2]", "test/cli/commands_test.py::test__cli__command_lint_parse[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command4]", "test/cli/commands_test.py::test__cli__command_lint_parse[command5]", "test/cli/commands_test.py::test__cli__command_lint_parse[command6]", "test/cli/commands_test.py::test__cli__command_lint_parse[command7]", "test/cli/commands_test.py::test__cli__command_lint_parse[command8]", "test/cli/commands_test.py::test__cli__command_lint_parse[command9]", "test/cli/commands_test.py::test__cli__command_lint_parse[command10]", "test/cli/commands_test.py::test__cli__command_lint_parse[command11]", "test/cli/commands_test.py::test__cli__command_lint_parse[command12]", "test/cli/commands_test.py::test__cli__command_lint_parse[command13]", "test/cli/commands_test.py::test__cli__command_lint_parse[command14]", "test/cli/commands_test.py::test__cli__command_lint_parse[command15]", "test/cli/commands_test.py::test__cli__command_lint_parse[command16]", "test/cli/commands_test.py::test__cli__command_lint_parse[command17]", "test/cli/commands_test.py::test__cli__command_lint_parse[command18]", "test/cli/commands_test.py::test__cli__command_lint_parse[command19]", "test/cli/commands_test.py::test__cli__command_lint_parse[command20]", "test/cli/commands_test.py::test__cli__command_lint_parse[command21]", "test/cli/commands_test.py::test__cli__command_lint_parse[command22]", "test/cli/commands_test.py::test__cli__command_lint_parse[command23]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command0-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command1-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command2-1]", "test/cli/commands_test.py::test__cli__command_lint_warning_explicit_file_ignored", "test/cli/commands_test.py::test__cli__command_lint_skip_ignore_files", "test/cli/commands_test.py::test__cli__command_lint_ignore_local_config", "test/cli/commands_test.py::test__cli__command_versioning", "test/cli/commands_test.py::test__cli__command_version", "test/cli/commands_test.py::test__cli__command_rules", "test/cli/commands_test.py::test__cli__command_dialects", "test/cli/commands_test.py::test__cli__command__fix[L001-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/whitespace_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L003-test/fixtures/linter/indentation_error_hard.sql]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_templating_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error_FIX_EVEN_UNPARSABLE]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[2_files_with_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-False]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[command-line-True]", "test/cli/commands_test.py::test_cli_fix_even_unparsable[config-file-False]", "test/cli/commands_test.py::test__cli__fix_loop_limit_behavior[--", "test/cli/commands_test.py::test__cli__command_fix_stdin[select", "test/cli/commands_test.py::test__cli__command_fix_stdin[", "test/cli/commands_test.py::test__cli__command_fix_stdin[SELECT", "test/cli/commands_test.py::test__cli__command_fix_stdin_logging_to_stderr", "test/cli/commands_test.py::test__cli__command_fix_stdin_safety", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[create", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[select", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-y-0-0]", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-n-65-1]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-json]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[select", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[SElect", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command0]", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command1]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation_native", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation-native]", "test/cli/commands_test.py::test___main___help", "test/cli/commands_test.py::test_encoding[utf-8-ascii]", "test/cli/commands_test.py::test_encoding[utf-8-sig-UTF-8-SIG]", "test/cli/commands_test.py::test_encoding[utf-32-UTF-32]", "test/cli/commands_test.py::test_cli_encoding[utf-8-command-line-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-command-line-True]", "test/cli/commands_test.py::test_cli_encoding[utf-8-config-file-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-config-file-True]", "test/cli/commands_test.py::test_cli_no_disable_noqa_flag", "test/cli/commands_test.py::test_cli_disable_noqa_flag", "test/cli/commands_test.py::test_cli_get_default_config", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_paths", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_files"], "environment_setup_commit": "8f6fd1d8a8d69b2c463fbcf5bd1131c47f12ad88"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-2419", "base_commit": "f1dba0e1dd764ae72d67c3d5e1471cf14d3db030", "patch": "diff --git a/src/sqlfluff/rules/L060.py b/src/sqlfluff/rules/L060.py\n--- a/src/sqlfluff/rules/L060.py\n+++ b/src/sqlfluff/rules/L060.py\n@@ -59,4 +59,8 @@ def _eval(self, context: RuleContext) -> Optional[LintResult]:\n ],\n )\n \n- return LintResult(context.segment, [fix])\n+ return LintResult(\n+ anchor=context.segment,\n+ fixes=[fix],\n+ description=f\"Use 'COALESCE' instead of '{context.segment.raw_upper}'.\",\n+ )\n", "test_patch": "diff --git a/test/rules/std_L060_test.py b/test/rules/std_L060_test.py\nnew file mode 100644\n--- /dev/null\n+++ b/test/rules/std_L060_test.py\n@@ -0,0 +1,12 @@\n+\"\"\"Tests the python routines within L060.\"\"\"\n+import sqlfluff\n+\n+\n+def test__rules__std_L060_raised() -> None:\n+ \"\"\"L060 is raised for use of ``IFNULL`` or ``NVL``.\"\"\"\n+ sql = \"SELECT\\n\\tIFNULL(NULL, 100),\\n\\tNVL(NULL,100);\"\n+ result = sqlfluff.lint(sql, rules=[\"L060\"])\n+\n+ assert len(result) == 2\n+ assert result[0][\"description\"] == \"Use 'COALESCE' instead of 'IFNULL'.\"\n+ assert result[1][\"description\"] == \"Use 'COALESCE' instead of 'NVL'.\"\n", "problem_statement": "Rule L060 could give a specific error message\nAt the moment rule L060 flags something like this:\r\n\r\n```\r\nL: 21 | P: 9 | L060 | Use 'COALESCE' instead of 'IFNULL' or 'NVL'.\r\n```\r\n\r\nSince we likely know the wrong word, it might be nice to actually flag that instead of both `IFNULL` and `NVL` - like most of the other rules do.\r\n\r\nThat is it should flag this:\r\n\r\n```\r\nL: 21 | P: 9 | L060 | Use 'COALESCE' instead of 'IFNULL'.\r\n```\r\n Or this:\r\n\r\n```\r\nL: 21 | P: 9 | L060 | Use 'COALESCE' instead of 'NVL'.\r\n```\r\n\r\nAs appropriate.\r\n\r\nWhat do you think @jpy-git ?\r\n\n", "hints_text": "@tunetheweb Yeah definitely, should be a pretty quick change \ud83d\ude0a", "created_at": "2022-01-22T12:21:52Z", "version": "0.8", "FAIL_TO_PASS": ["test/rules/std_L060_test.py::test__rules__std_L060_raised"], "PASS_TO_PASS": [], "environment_setup_commit": "a5c4eae4e3e419fe95460c9afd9cf39a35a470c4"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-3170", "base_commit": "6c026c76aa8b13eae54cd7e18d62b0a57fc71dce", "patch": "diff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py\n--- a/src/sqlfluff/cli/commands.py\n+++ b/src/sqlfluff/cli/commands.py\n@@ -247,7 +247,7 @@ def core_options(f: Callable) -> Callable:\n )(f)\n f = click.option(\n \"--encoding\",\n- default=\"autodetect\",\n+ default=None,\n help=(\n \"Specify encoding to use when reading and writing files. Defaults to \"\n \"autodetect.\"\n", "test_patch": "diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py\n--- a/test/cli/commands_test.py\n+++ b/test/cli/commands_test.py\n@@ -1256,44 +1256,40 @@ def test_encoding(encoding_in, encoding_out):\n )\n \n \n-def test_cli_pass_on_correct_encoding_argument():\n+@pytest.mark.parametrize(\n+ \"encoding,method,expect_success\",\n+ [\n+ (\"utf-8\", \"command-line\", False),\n+ (\"utf-8-SIG\", \"command-line\", True),\n+ (\"utf-8\", \"config-file\", False),\n+ (\"utf-8-SIG\", \"config-file\", True),\n+ ],\n+)\n+def test_cli_encoding(encoding, method, expect_success, tmpdir):\n \"\"\"Try loading a utf-8-SIG encoded file using the correct encoding via the cli.\"\"\"\n+ sql_path = \"test/fixtures/cli/encoding_test.sql\"\n+ if method == \"command-line\":\n+ options = [sql_path, \"--encoding\", encoding]\n+ else:\n+ assert method == \"config-file\"\n+ with open(str(tmpdir / \".sqlfluff\"), \"w\") as f:\n+ print(f\"[sqlfluff]\\ndialect=ansi\\nencoding = {encoding}\", file=f)\n+ shutil.copy(sql_path, tmpdir)\n+ options = [str(tmpdir / \"encoding_test.sql\")]\n result = invoke_assert_code(\n ret_code=65,\n args=[\n lint,\n- [\n- \"test/fixtures/cli/encoding_test.sql\",\n- \"--encoding\",\n- \"utf-8-SIG\",\n- ],\n- ],\n- )\n- raw_output = repr(result.output)\n-\n- # Incorrect encoding raises paring and lexer errors.\n- assert r\"L: 1 | P: 1 | LXR |\" not in raw_output\n- assert r\"L: 1 | P: 1 | PRS |\" not in raw_output\n-\n-\n-def test_cli_fail_on_wrong_encoding_argument():\n- \"\"\"Try loading a utf-8-SIG encoded file using the wrong encoding via the cli.\"\"\"\n- result = invoke_assert_code(\n- ret_code=65,\n- args=[\n- lint,\n- [\n- \"test/fixtures/cli/encoding_test.sql\",\n- \"--encoding\",\n- \"utf-8\",\n- ],\n+ options,\n ],\n )\n raw_output = repr(result.output)\n \n- # Incorrect encoding raises paring and lexer errors.\n- assert r\"L: 1 | P: 1 | LXR |\" in raw_output\n- assert r\"L: 1 | P: 1 | PRS |\" in raw_output\n+ # Incorrect encoding raises parsing and lexer errors.\n+ success1 = r\"L: 1 | P: 1 | LXR |\" not in raw_output\n+ success2 = r\"L: 1 | P: 1 | PRS |\" not in raw_output\n+ assert success1 == expect_success\n+ assert success2 == expect_success\n \n \n def test_cli_no_disable_noqa_flag():\n", "problem_statement": "Commented dash character converted to non utf-8 character\n### Search before asking\r\n\r\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\r\n\r\n\r\n### What Happened\r\n\r\nUpon fixing a query containing a multi-line comment, SQLFluff attempts to fix a commented line.\r\n\r\nThis:\r\n```sql\r\n/*\r\nTODO\r\n - tariff scenario \u2014> dm_tariff_scenario\r\n*/\r\n```\r\n\r\nBecame:\r\n```sql\r\n/*\r\nTODO\r\n - tariff scenario \u0097> dm_tariff_scenario\r\n*/\r\n``` \r\nThis in an invisible char represented as `<97>`\r\n\r\nThis causes an issue with dbt which can not compile with this char present\r\n\r\nNote this comment comes at the end of the file.\r\n\r\n### Expected Behaviour\r\n\r\nDoes not replace/fix anything that is commented\r\n\r\n### Observed Behaviour\r\n\r\n```bash\r\n $ sqlfluff fix dbt/models/marts/core/f_utility_statements.sql \r\n==== finding fixable violations ==== \r\n=== [dbt templater] Sorting Nodes... \r\n=== [dbt templater] Compiling dbt project... \r\n=== [dbt templater] Project Compiled. \r\n== [dbt/models/marts/core/f_utility_statements.sql] FAIL \r\nL: 1 | P: 5 | L001 | Unnecessary trailing whitespace. \r\nL: 2 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01] \r\nL: 3 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01] \r\nL: 4 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01] \r\nL: 4 | P: 6 | L019 | Found trailing comma. Expected only leading. \r\nL: 6 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01] \r\nL: 7 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01] \r\nL: 8 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01] \r\nL: 8 | P: 6 | L019 | Found trailing comma. Expected only leading. \r\nL: 10 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01] \r\nL: 11 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01] \r\nL: 12 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01] \r\nL: 12 | P: 6 | L019 | Found trailing comma. Expected only leading. \r\nL: 15 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01] \r\nL: 16 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01] [0/47960]\r\nL: 17 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 18 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 19 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 20 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 20 | P: 36 | L031 | Avoid aliases in from clauses and join conditions.\r\nL: 21 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 21 | P: 32 | L031 | Avoid aliases in from clauses and join conditions.\r\nL: 22 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01]\r\nL: 22 | P: 6 | L019 | Found trailing comma. Expected only leading.\r\nL: 24 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01]\r\nL: 26 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 26 | P: 15 | L001 | Unnecessary trailing whitespace.\r\nL: 27 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 28 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 29 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 30 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 31 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 32 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 32 | P: 24 | L011 | Implicit/explicit aliasing of table.\r\nL: 32 | P: 24 | L031 | Avoid aliases in from clauses and join conditions.\r\nL: 33 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 33 | P: 49 | L011 | Implicit/explicit aliasing of table.\r\nL: 33 | P: 49 | L031 | Avoid aliases in from clauses and join conditions.\r\nL: 33 | P: 52 | L001 | Unnecessary trailing whitespace.\r\nL: 34 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 36 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 37 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01]\r\nL: 37 | P: 6 | L019 | Found trailing comma. Expected only leading.\r\nL: 39 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01]\r\nL: 41 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 41 | P: 9 | L034 | Select wildcards then simple targets before calculations\r\n | and aggregates.\r\nL: 43 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 46 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 47 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 48 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 51 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 52 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 53 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 54 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 57 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 58 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 61 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 62 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 64 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 65 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 68 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 69 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 70 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 71 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 73 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 73 | P: 36 | L031 | Avoid aliases in from clauses and join conditions.\r\nL: 74 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 74 | P: 56 | L031 | Avoid aliases in from clauses and join conditions.\r\nL: 75 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 76 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 76 | P: 28 | L001 | Unnecessary trailing whitespace.\r\nL: 77 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 80 | P: 9 | L003 | Expected 0 indentations, found 2 [compared to line 01]\r\nL: 81 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 83 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 84 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01]\r\nL: 94 | P: 1 | L009 | Files must end with a single trailing newline.\r\n```\r\n\r\n### How to reproduce\r\n\r\n`sqlfluff fix` with provided `.sqlfluff` configuration\r\n\r\nSQL contains proprietary code and I am, likely, unable to provide a full snippet of the SQL \r\n\r\n### Dialect\r\n\r\nSnowflake\r\n\r\n### Version\r\n\r\n0.13.0 and 0.11.1\r\n\r\n### Configuration\r\n\r\n`.sqlfluff`:\r\n```\r\n[sqlfluff]\r\ntemplater = dbt\r\ndialect = snowflake\r\n\r\n[sqlfluff:templater:dbt]\r\nproject_dir = dbt/\r\n\r\n# Defaults on anything not specified explicitly: https://docs.sqlfluff.com/en/stable/configuration.html#default-configuration\r\n[sqlfluff:rules]\r\nmax_line_length = 120\r\ncomma_style = leading\r\n\r\n# Keyword capitalisation\r\n[sqlfluff:rules:L010]\r\ncapitalisation_policy = lower\r\n\r\n# TODO: this supports pascal but not snake\r\n# TODO: this inherits throwing violation on all unquoted identifiers... we can limit to aliases or column aliases\r\n# [sqlfluff:rules:L014]\r\n# extended_capitalisation_policy = pascal\r\n\r\n# TODO: not 100% certain that this default is correct\r\n# [sqlfluff:rules:L029]\r\n## Keywords should not be used as identifiers.\r\n# unquoted_identifiers_policy = aliases\r\n# quoted_identifiers_policy = none\r\n## Comma separated list of words to ignore for this rule\r\n# ignore_words = None\r\n\r\n# Function name capitalisation\r\n[sqlfluff:rules:L030]\r\nextended_capitalisation_policy = lower\r\n```\r\n\r\n### Are you willing to work on and submit a PR to address the issue?\r\n\r\n- [X] Yes I am willing to submit a PR!\r\n\r\n### Code of Conduct\r\n\r\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\r\n\n", "hints_text": "I can't reproduce this, but this usually happens when the file itself is in some other format, rather than UTF-8, to begin with. Can you confirm it's definitely UTF-8 encoded? some tips here on how to check this: https://stackoverflow.com/questions/6947749/how-to-check-if-a-txt-file-is-in-ascii-or-utf-8-format-in-windows-environment\nYou'll probably need to explicitly set the encoding. SQLFluff defaults to using `autodetect`, which is implemented by the third-party `chardet` package, but it's not guaranteed to always do the right thing. If it misbehaves, we (SQLFluff) can't do anything about it.\r\n\r\n```\r\n# can either be autodetect or a valid encoding e.g. utf-8, utf-8-sig\r\nencoding = autodetect\r\n```\r\n\r\nWe'd like to hear back from you, but this issue is likely to be closed as \"won't fix\"/\"can't fix\"\nI have confirmed that the file is indeed utf-8 encoded and I have explicitly set the encoding to utf-8 and retested with the same result.\r\n\r\nAfter running `sqlfluff fix` I have seen the encoding change from utf-8 to western (Windows 1252)\r\n\r\nEDIT: If i manually force the file to be utf-8 AFTER `sqlfluff fix`, it resolves the issue. Good to know, but not a sustainable solution\nDid you set the encoding property in `.sqlfluff`? It does not appear in the `.sqlfluff` file you provided above.\r\n\r\nAlso, please provide a test SQL file. You only provided a comment, not a complete file. When I run `sqlfluff fix` on the file, I get:\r\n```\r\n(sqlfluff-3.9.1) \u279c sqlfluff git:(main) \u2717 sqlfluff fix test.sql\r\n==== finding fixable violations ====\r\n==== no fixable linting violations found ====\r\nAll Finished \ud83d\udcdc \ud83c\udf89!\r\n```\nI did. The config file I provided does not contain it but I retested using your suggestion and had the same result\nI tried the above on my Mac. The resulting file looked okay to me:\r\n```\r\n - tariff scenario \u2014> dm_tariff_scenario\r\n```\r\n\r\nWhat operating system are using? Windows? Mac? Linux?\nI am on an intel mac with Montery 12.3.1\r\n\r\nAre you able to run sqlfluff fix twice in succession? The first run is fine, its the second run that fails\r\n\r\n(Depending on my editor, it may or may not show the offending character. ie vim shows it, sublime does not)\r\n\nYes, I can run it twice in succession. The first time, it fixes a bunch of things. The second time, no issues found. Partial output below.\r\n```\r\nL: 83 | P: 13 | L003 | Expected 0 indentations, found 3 [compared to line 01]\r\nL: 84 | P: 5 | L003 | Expected 0 indentations, found 1 [compared to line 01]\r\n==== fixing violations ====\r\n72 fixable linting violations found\r\nAre you sure you wish to attempt to fix these? [Y/n] ...\r\nAttempting fixes...\r\nPersisting Changes...\r\n== [test.sql] PASS\r\nDone. Please check your files to confirm.\r\nAll Finished \ud83d\udcdc \ud83c\udf89!\r\n [3 unfixable linting violations found]\r\n(sqlfluff-3.9.1) \u279c sqlfluff git:(main) \u2717 sqlfluff fix test.sql\r\n==== finding fixable violations ====\r\n==== no fixable linting violations found ====\r\nAll Finished \ud83d\udcdc \ud83c\udf89!\r\n [2 unfixable linting violations found]\r\n```\r\n\r\nI'm on an M1 Mac with Big Sur (11.5.2).\r\n\r\nVery strange behavior:\r\n* That I can't reproduce it on a similar machine\r\n* That setting `encoding = utf-8` in `.sqlfluff` doesn't fix it.\r\n\r\nNote that AFAIK, \"encoding\" is not a real property of most files file. It's a guess made when reading the file. Some file formats let you specify the encoding, but SQL is not one of them. Hence the need to use a package like `chardet`.\r\n\r\nE.g. Python lets you do it with a special comment: https://stackoverflow.com/questions/6289474/working-with-utf-8-encoding-in-python-source\nI just noticed interesting behavior. I ran with `-vv` to ensure my config and although I am specifying `encoding = utf-8`, the -vv output seems to suggest `autodetect`. It is honoring other config (like `dbt`). Attempting to see where I have gone wrong on my side\r\n\r\nEDIT: for context on directory structure: \r\n```\r\n.sqlfluff\r\n./dbt/models/marts/core/file.sql\r\n```\r\nI am running sqlfluff from the same directory as the `.sqlfluff` file ie `sqlfluff fix dbt/models/marts/core/file.sql`\nI've heard the behavior can become tricky if you have multiple .sqlfluff files in subdirectories, etc. Are you certain you added the setting in the correct section of the file? If you put it in the wrong place, it'll be ignored, and it'll use the default setting instead, which is autodetect.\nit is at the top level \r\n```\r\n[sqlfluff]\r\ntemplater = dbt\r\ndialect = snowflake\r\nencoding = utf-8\r\n\r\n...\r\n```\r\nas per your default configuration docs. There are no .sqlfluff files in sub folders in that directory\n@barrywhart Okay... so if I specify `--encoding utf-8` as a CLI command I am able to fix the file with no issue!! Thank you for helping with that!\r\n\r\nI am unsure why it is not honoring that config however. Is there a way you would recommend debugging this issue from my side? We use this both as a CLI tool and as a pre-commit - so we are able to use the `--encoding` option explicitly, but it provides peace of mind to know why it _seems_ to not honor specific configs\r\n\r\nI have changed other configs (ie adding an `excluded_rule`) and it IS honoring that (with no other changes to how i am running it)\r\n\r\nAlso super appreciate all the help :) \nLet me look into it later (probably in the next day or two). Not many people use this option, so I'd like to double check that it's being read correctly from config.\nawesome! I appreciate it @barrywhart (and @tunetheweb )!\r\n\r\nWe, as an organization, are investing in SQLFluff as our production linter and we appreciate your support!\nThanks for the kind words. It's exciting to us seeing the project catching on. I've been involved with the project since late 2019, and I'm proud of the progress it's made. It seems to be becoming pretty mainstream now. One reason I've stayed involved is, how often do you get to help invent a fundamental new industry tool? \ud83d\ude0a\r\n\r\nBTW, feel free to delete your example SQL from the issue. It seems like we may not need it anymore?\nExactly! I have been loosely following this project for the past year and have been pushing to use it widely for a while! We adopted DBT and, since SQLFluff interacts well with DBT, we got the buy-in to invest :)\r\n\r\nAnd yes I will delete the SQL!\r\n\r\nPlease let me know what you find relating to the encoding configuration! I am continuing to fiddle from my side!\nI'm seeing the same issue -- seems that the `encoding` setting in `.sqlfluff` is not being read correctly:\r\n```\r\n[sqlfluff]\r\nencoding = utf-8\r\n```\r\n\r\nWe have automated tests for encoding, but they are lower-level tests (i.e. they exercise internal code directly, not reading encoding from `.sqlfluff`).\r\n\r\nI'll take a closer look. Presumably, this should be easy to fix.", "created_at": "2022-04-24T21:45:11Z", "version": "0.12", "FAIL_TO_PASS": ["test/cli/commands_test.py::test_cli_encoding[utf-8-config-file-False]"], "PASS_TO_PASS": ["test/cli/commands_test.py::test__cli__command_directed", "test/cli/commands_test.py::test__cli__command_dialect", "test/cli/commands_test.py::test__cli__command_dialect_legacy", "test/cli/commands_test.py::test__cli__command_extra_config_fail", "test/cli/commands_test.py::test__cli__command_lint_stdin[command0]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command1]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command2]", "test/cli/commands_test.py::test__cli__command_lint_stdin[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command0]", "test/cli/commands_test.py::test__cli__command_lint_parse[command1]", "test/cli/commands_test.py::test__cli__command_lint_parse[command2]", "test/cli/commands_test.py::test__cli__command_lint_parse[command3]", "test/cli/commands_test.py::test__cli__command_lint_parse[command4]", "test/cli/commands_test.py::test__cli__command_lint_parse[command5]", "test/cli/commands_test.py::test__cli__command_lint_parse[command6]", "test/cli/commands_test.py::test__cli__command_lint_parse[command7]", "test/cli/commands_test.py::test__cli__command_lint_parse[command8]", "test/cli/commands_test.py::test__cli__command_lint_parse[command9]", "test/cli/commands_test.py::test__cli__command_lint_parse[command10]", "test/cli/commands_test.py::test__cli__command_lint_parse[command11]", "test/cli/commands_test.py::test__cli__command_lint_parse[command12]", "test/cli/commands_test.py::test__cli__command_lint_parse[command13]", "test/cli/commands_test.py::test__cli__command_lint_parse[command14]", "test/cli/commands_test.py::test__cli__command_lint_parse[command15]", "test/cli/commands_test.py::test__cli__command_lint_parse[command16]", "test/cli/commands_test.py::test__cli__command_lint_parse[command17]", "test/cli/commands_test.py::test__cli__command_lint_parse[command18]", "test/cli/commands_test.py::test__cli__command_lint_parse[command19]", "test/cli/commands_test.py::test__cli__command_lint_parse[command20]", "test/cli/commands_test.py::test__cli__command_lint_parse[command21]", "test/cli/commands_test.py::test__cli__command_lint_parse[command22]", "test/cli/commands_test.py::test__cli__command_lint_parse[command23]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command0-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command1-1]", "test/cli/commands_test.py::test__cli__command_lint_parse_with_retcode[command2-1]", "test/cli/commands_test.py::test__cli__command_lint_warning_explicit_file_ignored", "test/cli/commands_test.py::test__cli__command_lint_skip_ignore_files", "test/cli/commands_test.py::test__cli__command_lint_ignore_local_config", "test/cli/commands_test.py::test__cli__command_versioning", "test/cli/commands_test.py::test__cli__command_version", "test/cli/commands_test.py::test__cli__command_rules", "test/cli/commands_test.py::test__cli__command_dialects", "test/cli/commands_test.py::test__cli__command__fix[L001-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/whitespace_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L008-test/fixtures/linter/indentation_errors.sql]", "test/cli/commands_test.py::test__cli__command__fix[L003-test/fixtures/linter/indentation_error_hard.sql]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_templating_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[0_lint_errors_1_suppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[1_lint_error_1_unsuppressed_parse_error_FIX_EVEN_UNPARSABLE]", "test/cli/commands_test.py::test__cli__fix_error_handling_behavior[2_files_with_lint_errors_1_unsuppressed_parse_error]", "test/cli/commands_test.py::test__cli__fix_loop_limit_behavior[--", "test/cli/commands_test.py::test__cli__command_fix_stdin[select", "test/cli/commands_test.py::test__cli__command_fix_stdin[", "test/cli/commands_test.py::test__cli__command_fix_stdin[SELECT", "test/cli/commands_test.py::test__cli__command_fix_stdin_logging_to_stderr", "test/cli/commands_test.py::test__cli__command_fix_stdin_safety", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[create", "test/cli/commands_test.py::test__cli__command_fix_stdin_error_exit_code[select", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-y-0-0]", "test/cli/commands_test.py::test__cli__command__fix_no_force[L001-test/fixtures/linter/indentation_errors.sql-n-65-1]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[None-json]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_parse_serialize_from_stdin[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[select", "test/cli/commands_test.py::test__cli__command_lint_serialize_from_stdin[SElect", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command0]", "test/cli/commands_test.py::test__cli__command_fail_nice_not_found[command1]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[None-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-human]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-yaml]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-json]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_multiple_files[outfile-github-annotation-native]", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation", "test/cli/commands_test.py::test__cli__command_lint_serialize_github_annotation_native", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation]", "test/cli/commands_test.py::test__cli__command_lint_serialize_annotation_level_error_failure_equivalent[github-annotation-native]", "test/cli/commands_test.py::test___main___help", "test/cli/commands_test.py::test_encoding[utf-8-ascii]", "test/cli/commands_test.py::test_encoding[utf-8-sig-UTF-8-SIG]", "test/cli/commands_test.py::test_encoding[utf-32-UTF-32]", "test/cli/commands_test.py::test_cli_encoding[utf-8-command-line-False]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-command-line-True]", "test/cli/commands_test.py::test_cli_encoding[utf-8-SIG-config-file-True]", "test/cli/commands_test.py::test_cli_no_disable_noqa_flag", "test/cli/commands_test.py::test_cli_disable_noqa_flag", "test/cli/commands_test.py::test_cli_get_default_config", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_disabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_paths", "test/cli/commands_test.py::TestProgressBars::test_cli_lint_enabled_progress_bar_multiple_files"], "environment_setup_commit": "8f6fd1d8a8d69b2c463fbcf5bd1131c47f12ad88"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-2846", "base_commit": "f37dc1410cefc4e08ed8110f820c9071bc4b0c7d", "patch": "diff --git a/src/sqlfluff/core/templaters/slicers/tracer.py b/src/sqlfluff/core/templaters/slicers/tracer.py\n--- a/src/sqlfluff/core/templaters/slicers/tracer.py\n+++ b/src/sqlfluff/core/templaters/slicers/tracer.py\n@@ -185,6 +185,23 @@ def next_slice_id(self) -> str:\n self.slice_id += 1\n return result\n \n+ def slice_info_for_literal(self, length, prefix=\"\") -> RawSliceInfo:\n+ \"\"\"Returns a RawSliceInfo for a literal.\n+\n+ In the alternate template, literals are replaced with a uniquely\n+ numbered, easily-to-parse literal. JinjaTracer uses this output as\n+ a \"breadcrumb trail\" to deduce the execution path through the template.\n+\n+ This is important even if the original literal (i.e. in the raw SQL\n+ file) was empty, as is the case when Jinja whitespace control is used\n+ (e.g. \"{%- endif -%}\"), because fewer breadcrumbs means JinjaTracer has\n+ to *guess* the path, in which case it assumes simple, straight-line\n+ execution, which can easily be wrong with loops and conditionals.\n+ \"\"\"\n+ unique_alternate_id = self.next_slice_id()\n+ alternate_code = f\"\\0{prefix}{unique_alternate_id}_{length}\"\n+ return RawSliceInfo(unique_alternate_id, alternate_code, [])\n+\n def _slice_template(self) -> List[RawFileSlice]:\n \"\"\"Slice template in jinja.\n \n@@ -217,12 +234,6 @@ def _slice_template(self) -> List[RawFileSlice]:\n for _, elem_type, raw in self.env.lex(self.raw_str):\n # Replace literal text with a unique ID.\n if elem_type == \"data\":\n- if set_idx is None:\n- unique_alternate_id = self.next_slice_id()\n- alternate_code = f\"\\0{unique_alternate_id}_{len(raw)}\"\n- else:\n- unique_alternate_id = self.next_slice_id()\n- alternate_code = f\"\\0set{unique_alternate_id}_{len(raw)}\"\n result.append(\n RawFileSlice(\n raw,\n@@ -230,8 +241,8 @@ def _slice_template(self) -> List[RawFileSlice]:\n idx,\n )\n )\n- self.raw_slice_info[result[-1]] = RawSliceInfo(\n- unique_alternate_id, alternate_code, []\n+ self.raw_slice_info[result[-1]] = self.slice_info_for_literal(\n+ len(raw), \"\" if set_idx is None else \"set\"\n )\n idx += len(raw)\n continue\n@@ -274,7 +285,7 @@ def _slice_template(self) -> List[RawFileSlice]:\n )\n # Treat the skipped whitespace as a literal.\n result.append(RawFileSlice(skipped_str, \"literal\", idx))\n- self.raw_slice_info[result[-1]] = RawSliceInfo(\"\", \"\", [])\n+ self.raw_slice_info[result[-1]] = self.slice_info_for_literal(0)\n idx += num_chars_skipped\n \n # raw_end and raw_begin behave a little differently in\n@@ -354,10 +365,6 @@ def _slice_template(self) -> List[RawFileSlice]:\n # returns, it has simply grouped them differently than we\n # want.\n trailing_chars = len(m.group(0))\n- if block_type.startswith(\"block_\"):\n- alternate_code = self._remove_block_whitespace_control(\n- str_buff[:-trailing_chars]\n- )\n result.append(\n RawFileSlice(\n str_buff[:-trailing_chars],\n@@ -378,11 +385,9 @@ def _slice_template(self) -> List[RawFileSlice]:\n idx,\n )\n )\n- self.raw_slice_info[result[-1]] = RawSliceInfo(\"\", \"\", [])\n+ self.raw_slice_info[result[-1]] = self.slice_info_for_literal(0)\n idx += trailing_chars\n else:\n- if block_type.startswith(\"block_\"):\n- alternate_code = self._remove_block_whitespace_control(str_buff)\n result.append(\n RawFileSlice(\n str_buff,\n@@ -424,17 +429,3 @@ def _slice_template(self) -> List[RawFileSlice]:\n stack.pop()\n str_buff = \"\"\n return result\n-\n- @classmethod\n- def _remove_block_whitespace_control(cls, in_str: str) -> Optional[str]:\n- \"\"\"Removes whitespace control from a Jinja block start or end.\n-\n- Use of Jinja whitespace stripping (e.g. `{%-` or `-%}`) causes the\n- template to produce less output. This makes JinjaTracer's job harder,\n- because it uses the \"bread crumb trail\" of output to deduce the\n- execution path through the template. This change has no impact on the\n- actual Jinja output, which uses the original, unmodified code.\n- \"\"\"\n- result = regex.sub(r\"^{%-\", \"{%\", in_str)\n- result = regex.sub(r\"-%}$\", \"%}\", result)\n- return result if result != in_str else None\n", "test_patch": "diff --git a/test/core/templaters/jinja_test.py b/test/core/templaters/jinja_test.py\n--- a/test/core/templaters/jinja_test.py\n+++ b/test/core/templaters/jinja_test.py\n@@ -756,7 +756,6 @@ def test__templater_jinja_slice_template(test, result):\n (\"block_start\", slice(0, 25, None), slice(0, 0, None)),\n (\"literal\", slice(25, 30, None), slice(0, 5, None)),\n (\"block_start\", slice(30, 47, None), slice(5, 5, None)),\n- (\"literal\", slice(47, 67, None), slice(5, 5, None)),\n (\"block_end\", slice(67, 78, None), slice(5, 5, None)),\n (\"literal\", slice(78, 79, None), slice(5, 5, None)),\n (\"block_end\", slice(79, 92, None), slice(5, 5, None)),\n@@ -796,6 +795,57 @@ def test__templater_jinja_slice_template(test, result):\n (\"literal\", slice(46, 57, None), slice(11, 22, None)),\n ],\n ),\n+ (\n+ # Test for issue 2786. Also lots of whitespace control. In this\n+ # case, removing whitespace control alone wasn't enough. In order\n+ # to get a good trace, JinjaTracer had to be updated so the\n+ # alternate template included output for the discarded whitespace.\n+ \"\"\"select\n+ id,\n+ {%- for features in [\"value4\", \"value5\"] %}\n+ {%- if features in [\"value7\"] %}\n+ {{features}}\n+ {%- if not loop.last -%},{% endif %}\n+ {%- else -%}\n+ {{features}}\n+ {%- if not loop.last -%},{% endif %}\n+ {%- endif -%}\n+ {%- endfor %}\n+from my_table\n+\"\"\",\n+ None,\n+ [\n+ (\"literal\", slice(0, 14, None), slice(0, 14, None)),\n+ (\"literal\", slice(14, 19, None), slice(14, 14, None)),\n+ (\"block_start\", slice(19, 62, None), slice(14, 14, None)),\n+ (\"literal\", slice(62, 71, None), slice(14, 14, None)),\n+ (\"block_start\", slice(71, 103, None), slice(14, 14, None)),\n+ (\"block_mid\", slice(186, 198, None), slice(14, 14, None)),\n+ (\"literal\", slice(198, 211, None), slice(14, 14, None)),\n+ (\"templated\", slice(211, 223, None), slice(14, 20, None)),\n+ (\"literal\", slice(223, 236, None), slice(20, 20, None)),\n+ (\"block_start\", slice(236, 260, None), slice(20, 20, None)),\n+ (\"literal\", slice(260, 261, None), slice(20, 21, None)),\n+ (\"block_end\", slice(261, 272, None), slice(21, 21, None)),\n+ (\"literal\", slice(272, 281, None), slice(21, 21, None)),\n+ (\"block_end\", slice(281, 294, None), slice(21, 21, None)),\n+ (\"literal\", slice(294, 299, None), slice(21, 21, None)),\n+ (\"block_end\", slice(299, 312, None), slice(21, 21, None)),\n+ (\"literal\", slice(62, 71, None), slice(21, 21, None)),\n+ (\"block_start\", slice(71, 103, None), slice(21, 21, None)),\n+ (\"block_mid\", slice(186, 198, None), slice(21, 21, None)),\n+ (\"literal\", slice(198, 211, None), slice(21, 21, None)),\n+ (\"templated\", slice(211, 223, None), slice(21, 27, None)),\n+ (\"literal\", slice(223, 236, None), slice(27, 27, None)),\n+ (\"block_start\", slice(236, 260, None), slice(27, 27, None)),\n+ (\"block_end\", slice(261, 272, None), slice(27, 27, None)),\n+ (\"literal\", slice(272, 281, None), slice(27, 27, None)),\n+ (\"block_end\", slice(281, 294, None), slice(27, 27, None)),\n+ (\"literal\", slice(294, 299, None), slice(27, 27, None)),\n+ (\"block_end\", slice(299, 312, None), slice(27, 27, None)),\n+ (\"literal\", slice(312, 327, None), slice(27, 42, None)),\n+ ],\n+ ),\n ],\n )\n def test__templater_jinja_slice_file(raw_file, override_context, result, caplog):\n", "problem_statement": "ValueError: Position Not Found for lint/parse/fix, not clear why\n### Search before asking\r\n\r\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\r\n\r\n\r\n### What Happened\r\n\r\nI have admittedly messy dbt sql model that gets the following error when I try to lint, parse or fix it with sqlfluff - every other model can be processed using the same settings, but this one throws the same error below even if I only run a single rule e.g. L009.\r\n\r\nUnfortunately I cannot share the model itself but I can describe some notable features:\r\n- begins with a dbt incremental config\r\n- then sets three variables, each a list of strings\r\n- Has two `for` loops with nested `if` conditions\r\n- Has one very long line doing arithmetic operations involving both hardcoded values and columns from a two joined CTEs\r\n\r\n### Expected Behaviour\r\n\r\nNot the above error\r\n\r\n### Observed Behaviour\r\n\r\n```\r\nWARNING Unable to lint models/ltv_prediction_model/ltv_prediction.sql due to an internal error. Please report this as an issue w\r\nith your query's contents and stacktrace below!\r\nTo hide this warning, add the failing file to .sqlfluffignore\r\nTraceback (most recent call last):\r\n File \"/Users/dlyons/.pyenv/versions/3.9.4/lib/python3.9/site-packages/sqlfluff/core/linter/runner.py\", line 103, in run\r\n yield partial()\r\n File \"/Users/dlyons/.pyenv/versions/3.9.4/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py\", line 666, in lint_rendered\r\n parsed = cls.parse_rendered(rendered)\r\n File \"/Users/dlyons/.pyenv/versions/3.9.4/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py\", line 352, in parse_rendere\r\n\r\nd\r\n tokens, lvs, config = cls._lex_templated_file(\r\n File \"/Users/dlyons/.pyenv/versions/3.9.4/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py\", line 139, in _lex_template\r\nd_file\r\n tokens, lex_vs = lexer.lex(templated_file)\r\n File \"/Users/dlyons/.pyenv/versions/3.9.4/lib/python3.9/site-packages/sqlfluff/core/parser/lexer.py\", line 321, in lex\r\n segments: Tuple[RawSegment, ...] = self.elements_to_segments(\r\n File \"/Users/dlyons/.pyenv/versions/3.9.4/lib/python3.9/site-packages/sqlfluff/core/parser/lexer.py\", line 348, in elements_to_se\r\ngments\r\n source_slice = templated_file.templated_slice_to_source_slice(\r\n File \"/Users/dlyons/.pyenv/versions/3.9.4/lib/python3.9/site-packages/sqlfluff/core/templaters/base.py\", line 294, in templated_s\r\nlice_to_source_slice\r\n ts_stop_sf_start, ts_stop_sf_stop = self._find_slice_indices_of_templated_pos(\r\n File \"/Users/dlyons/.pyenv/versions/3.9.4/lib/python3.9/site-packages/sqlfluff/core/templaters/base.py\", line 180, in _find_slice\r\n_indices_of_templated_pos\r\n raise ValueError(\"Position Not Found\")\r\nValueError: Position Not Found\r\n```\r\n\r\n### How to reproduce\r\n\r\n```\r\n{{\r\n config(\r\n materialized='incremental',\r\n unique_key='md5_surrogate_key_main'\r\n )\r\n}}\r\n\r\n{%- set first_list = [\"value1\", \"value2\", \"value3\"] -%}\r\n{%- set second_list = [\"value4\", \"value5\", \"value6\"] -%}\r\n{%- set third_list = [\"value7\", \"value8\", \"value9\"] -%}\r\n\r\nwith fill_na_values as (\r\n select\r\n id,\r\n run_date,\r\n md5_surrogate_key_main,\r\n {%- for features in second_list %}\r\n {%- if features in third_list %}\r\n coalesce({{features}}, (select feature_mode from {{ ref('second_list') }} where features = '{{features}}')) as {{features}}\r\n {%- if not loop.last -%},{% endif %}\r\n {%- else -%}\r\n coalesce({{features}}, (select feature_mean from {{ ref('second_list') }} where features = '{{features}}')) as {{features}}\r\n {%- if not loop.last -%},{% endif %}\r\n {%- endif -%}\r\n {%- endfor %}\r\n from {{ ref('training_dataset') }}\r\n {%- if is_incremental() %}\r\n where current_date >= (select max(run_date) from {{ this }})\r\n {%- else %}\r\n where run_date >= '2021-01-01'\r\n {%- endif %}\r\n),\r\n\r\nwinsorize_data as (\r\n select\r\n md5_surrogate_key_main,\r\n {%- for features in second_list %}\r\n {%- if features in first_list %}\r\n case\r\n when {{features}} < (select fifth_percentile from {{ ref('first_list') }} where winsorize_column = '{{features}}')\r\n then (select fifth_percentile from {{ ref('first_list') }} where winsorize_column = '{{features}}')\r\n when {{features}} > (select ninetyfifth_percentile from {{ ref('first_list') }} where winsorize_column = '{{features}}')\r\n then (select ninetyfifth_percentile from {{ ref('first_list') }} where winsorize_column = '{{features}}')\r\n else {{features}}\r\n end as {{features}}\r\n {%- if not loop.last -%},{% endif %}\r\n {%- else %}\r\n {{features}}\r\n {%- if not loop.last -%},{% endif %}\r\n {%- endif %}\r\n {%- endfor %}\r\n from fill_na_values\r\n),\r\n\r\nscaling_data as (\r\n select\r\n md5_surrogate_key_main,\r\n {%- for features in second_list %}\r\n ({{features}} - (select feature_mean from {{ ref('second_list') }} where features = '{{features}}'))/(select feature_std from {{ ref('second_list') }} where features = '{{features}}') as {{features}}\r\n {%- if not loop.last -%},{% endif %}\r\n {%- endfor %}\r\n from winsorize_data\r\n),\r\n\r\napply_ceofficients as (\r\n select\r\n md5_surrogate_key_main,\r\n {%- for features in second_list %}\r\n {{features}} * (select coefficients from {{ ref('second_list') }} where features = '{{features}}') as {{features}}_coef\r\n {%- if not loop.last -%},{% endif %}\r\n {%- endfor %}\r\n from scaling_data\r\n),\r\n\r\nlogistic_prediction as (\r\n select\r\n fan.*,\r\n 1/(1+EXP(-(0.24602303+coef1+coef2+coef3+coef4+coef5+coef6+coef7+coef8+coef9+available_balance_coef+coef10+coef11+coef12+coef13+coef14))) as prediction_probability,\r\n case when prediction_probability < .5 then 0 else 1 end as prediction_class\r\n from apply_ceofficients ac\r\n inner join fill_na_values fan\r\n on ac.md5_surrogate_key_main = fan.md5_surrogate_key_main\r\n)\r\n\r\nselect * from logistic_prediction\r\n```\r\n\r\n### Dialect\r\n\r\nSnowflake\r\n\r\n### Version\r\n\r\n0.10.1\r\n\r\n### Configuration\r\n\r\n```\r\n[sqlfluff]\r\n# verbose is an integer (0-2) indicating the level of log output\r\nverbose = 0\r\n# Turn off color formatting of output\r\nnocolor = False\r\ndialect = snowflake\r\ntemplater = jinja\r\n# Comma separated list of rules to check, or None for all\r\nrules = L001,L002,L003,L004,L005,L009,L010,L013,L014,L015,L017,L018,L019,L020,L021,L022,L023,L024,L026,L027,L028,L030,L036,L037,L038,L039,L040,L044,L045,L046,L050,L051,L058,L061\r\n# Comma separated list of rules to exclude, or None\r\nexclude_rules = L006,L008,L011,L012,L025,L029,L031,L034,L035,L041,L042,L043,L052\r\n# The depth to recursively parse to (0 for unlimited)\r\nrecurse = 0\r\n# Below controls SQLFluff output, see max_line_length for SQL output\r\noutput_line_length = 80\r\n# Number of passes to run before admitting defeat\r\nrunaway_limit = 10\r\n# Ignore errors by category (one or more of the following, separated by commas: lexing,linting,parsing,templating)\r\nignore = None\r\n# Ignore linting errors found within sections of code coming directly from\r\n# templated code (e.g. from within Jinja curly braces. Note that it does not\r\n# ignore errors from literal code found within template loops.\r\nignore_templated_areas = True\r\n# can either be autodetect or a valid encoding e.g. utf-8, utf-8-sig\r\nencoding = autodetect\r\n# Ignore inline overrides (e.g. to test if still required)\r\ndisable_noqa = False\r\n# Comma separated list of file extensions to lint\r\n# NB: This config will only apply in the root folder\r\nsql_file_exts = .sql,.sql.j2,.dml,.ddl\r\n# Allow fix to run on files, even if they contain parsing errors\r\n# Note altering this is NOT RECOMMENDED as can corrupt SQL\r\nfix_even_unparsable = False\r\n\r\n[sqlfluff:indentation]\r\n# See https://docs.sqlfluff.com/en/stable/indentation.html\r\nindented_joins = False\r\nindented_ctes = False\r\nindented_using_on = True\r\ntemplate_blocks_indent = True\r\n\r\n[sqlfluff:templater]\r\nunwrap_wrapped_queries = True\r\n\r\n[sqlfluff:templater:jinja]\r\napply_dbt_builtins = True\r\n\r\n[sqlfluff:templater:jinja:macros]\r\n# Macros provided as builtins for dbt projects\r\ndbt_ref = {% macro ref(model_ref) %}{{model_ref}}{% endmacro %}\r\ndbt_source = {% macro source(source_name, table) %}{{source_name}}_{{table}}{% endmacro %}\r\ndbt_config = {% macro config() %}{% for k in kwargs %}{% endfor %}{% endmacro %}\r\ndbt_var = {% macro var(variable, default='') %}item{% endmacro %}\r\ndbt_is_incremental = {% macro is_incremental() %}True{% endmacro %}\r\n\r\n# Some rules can be configured directly from the config common to other rules\r\n[sqlfluff:rules]\r\ntab_space_size = 4\r\nmax_line_length = 80\r\nindent_unit = space\r\ncomma_style = trailing\r\nallow_scalar = True\r\nsingle_table_references = consistent\r\nunquoted_identifiers_policy = all\r\n\r\n# Some rules have their own specific config\r\n[sqlfluff:rules:L007]\r\noperator_new_lines = after\r\n\r\n[sqlfluff:rules:L010]\r\n# Keywords\r\ncapitalisation_policy = consistent\r\n# Comma separated list of words to ignore for this rule\r\nignore_words = None\r\n\r\n[sqlfluff:rules:L011]\r\n# Aliasing preference for tables\r\naliasing = explicit\r\n\r\n[sqlfluff:rules:L012]\r\n# Aliasing preference for columns\r\naliasing = explicit\r\n\r\n[sqlfluff:rules:L014]\r\n# Unquoted identifiers\r\nextended_capitalisation_policy = consistent\r\n# Comma separated list of words to ignore for this rule\r\nignore_words = None\r\n\r\n[sqlfluff:rules:L016]\r\n# Line length\r\nignore_comment_lines = False\r\nignore_comment_clauses = False\r\n\r\n[sqlfluff:rules:L026]\r\n# References must be in FROM clause\r\n# Disabled for some dialects (e.g. bigquery)\r\nforce_enable = False\r\n\r\n[sqlfluff:rules:L028]\r\n# References must be consistently used\r\n# Disabled for some dialects (e.g. bigquery)\r\nforce_enable = False\r\n\r\n[sqlfluff:rules:L029]\r\n# Keywords should not be used as identifiers.\r\nunquoted_identifiers_policy = aliases\r\nquoted_identifiers_policy = none\r\n# Comma separated list of words to ignore for this rule\r\nignore_words = None\r\n\r\n[sqlfluff:rules:L030]\r\n# Function names\r\ncapitalisation_policy = consistent\r\n# Comma separated list of words to ignore for this rule\r\nignore_words = None\r\n\r\n[sqlfluff:rules:L038]\r\n# Trailing commas\r\nselect_clause_trailing_comma = forbid\r\n\r\n[sqlfluff:rules:L040]\r\n# Null & Boolean Literals\r\ncapitalisation_policy = consistent\r\n# Comma separated list of words to ignore for this rule\r\nignore_words = None\r\n\r\n[sqlfluff:rules:L042]\r\n# By default, allow subqueries in from clauses, but not join clauses\r\nforbid_subquery_in = join\r\n\r\n[sqlfluff:rules:L047]\r\n# Consistent syntax to count all rows\r\nprefer_count_1 = False\r\nprefer_count_0 = False\r\n\r\n[sqlfluff:rules:L052]\r\n# Semi-colon formatting approach\r\nmultiline_newline = False\r\nrequire_final_semicolon = False\r\n\r\n[sqlfluff:rules:L054]\r\n# GROUP BY/ORDER BY column references\r\ngroup_by_and_order_by_style = consistent\r\n\r\n[sqlfluff:rules:L057]\r\n# Special characters in identifiers\r\nunquoted_identifiers_policy = all\r\nquoted_identifiers_policy = all\r\nallow_space_in_identifier = False\r\nadditional_allowed_characters = \"\"\r\n\r\n[sqlfluff:rules:L059]\r\n# Policy on quoted and unquoted identifiers\r\nprefer_quoted_identifiers = False\r\n\r\n[sqlfluff:rules:L062]\r\n# Comma separated list of blocked words that should not be used\r\nblocked_words = None\r\n\r\n### Are you willing to work on and submit a PR to address the issue?\r\n\r\n- [X] Yes I am willing to submit a PR!\r\n\r\n### Code of Conduct\r\n\r\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\r\n```\n", "hints_text": "> Proprietary concerns prevent me from sharing the query itself, I could try to boil it down to a mock version that replicates the error.\r\n\r\nThis would be needed before we can make any progress on this I'm afraid. The stack trace is good, and will help us identify the area of code, but without the SQL we can't know why and this issue will need to be closed.\n@tunetheweb fair play, let me try to create a stripped down representation.\n@tunetheweb there you go!\nI managed to reproduce this in the latest `main`. Taking a quick look...\nIt fails when looking for templated position 198. The highest-numbered position in the `sliced_file` collection is 190. \r\n```\r\n(Pdb) templated_pos\r\n198\r\n(Pdb) pp self.sliced_file\r\n[TemplatedFileSlice(slice_type='templated', source_slice=slice(0, 103, None), templated_slice=slice(0, 0, None)),\r\n TemplatedFileSlice(slice_type='literal', source_slice=slice(103, 105, None), templated_slice=slice(0, 0, None)),\r\n TemplatedFileSlice(slice_type='block_start', source_slice=slice(105, 160, None), templated_slice=slice(0, 0, None)),\r\n...\r\n TemplatedFileSlice(slice_type='literal', source_slice=slice(2822, 2823, None), templated_slice=slice(190, 190, None)),\r\n TemplatedFileSlice(slice_type='block_end', source_slice=slice(2823, 2834, None), templated_slice=slice(190, 190, None)),\r\n TemplatedFileSlice(slice_type='literal', source_slice=slice(2834, 2843, None), templated_slice=slice(190, 190, None)),\r\n TemplatedFileSlice(slice_type='block_end', source_slice=slice(2843, 2856, None), templated_slice=slice(190, 190, None)),\r\n TemplatedFileSlice(slice_type='literal', source_slice=slice(2856, 3358, None), templated_slice=slice(190, 190, None))]\r\n(Pdb) \r\n```\r\n\r\nThe `sliced_file` is clearly wrong, because the rendered SQL is 2,083 characters long:\r\n```\r\n(Pdb) len(str(self))\r\n2083\r\n```\nThe templater is losing track of things at line 19 of the input file:\r\n```\r\n coalesce({{features}}, (select feature_mode from {{ ref('second_list') }} where features = '{{features}}')) as {{features}}\r\n```\r\n\r\nPosition 198 is where the code `{{features}}` renders, just after `coalesce(`.\nThe following simpler SQL can be used to reproduce the same issue:\r\n```\r\nselect\r\n {%- for features in [\"value4\", \"value5\"] %}\r\n {%- if features in [\"value7\"] %}\r\n {{features}}\r\n {%- if not loop.last -%},{% endif %}\r\n {%- else -%}\r\n {{features}}\r\n {%- if not loop.last -%},{% endif %}\r\n {%- endif -%}\r\n {%- endfor %}\r\nfrom my_table\r\n```\r\n\r\nThis is another test case I extracted (may be the same bug, not sure):\r\n```\r\n{%- set first_list = [\"value1\", \"value2\", \"value3\"] -%}\r\n{%- set second_list = [\"value4\", \"value5\", \"value6\"] -%}\r\n\r\nwith winsorize_data as (\r\n select\r\n md5_surrogate_key_main,\r\n {%- for features in second_list %}\r\n {%- if features in first_list %}\r\n case\r\n when {{features}} < (select fifth_percentile from {{ ref('first_list') }} where winsorize_column = '{{features}}')\r\n then (select fifth_percentile from {{ ref('first_list') }} where winsorize_column = '{{features}}')\r\n when {{features}} > (select ninetyfifth_percentile from {{ ref('first_list') }} where winsorize_column = '{{features}}')\r\n then (select ninetyfifth_percentile from {{ ref('first_list') }} where winsorize_column = '{{features}}')\r\n else {{features}}\r\n end as {{features}}\r\n {%- if not loop.last -%},{% endif %}\r\n {%- else %}\r\n {{features}}\r\n {%- if not loop.last -%},{% endif %}\r\n {%- endif %}\r\n {%- endfor %}\r\n from ref('training_dataset')\r\n),\r\n\r\nscaling_data as (\r\n select\r\n md5_surrogate_key_main,\r\n {%- for features in second_list %}\r\n ({{features}} - (select feature_mean from {{ ref('second_list') }} where features = '{{features}}'))/(select feature_std from {{ ref('second_list') }} where features = '{{features}}') as {{features}}\r\n {%- if not loop.last -%},{% endif %}\r\n {%- endfor %}\r\n from winsorize_data\r\n),\r\n\r\napply_ceofficients as (\r\n select\r\n md5_surrogate_key_main,\r\n {%- for features in second_list %}\r\n {{features}} * (select coefficients from {{ ref('second_list') }} where features = '{{features}}') as {{features}}_coef\r\n {%- if not loop.last -%},{% endif %}\r\n {%- endfor %}\r\n from scaling_data\r\n),\r\n\r\nlogistic_prediction as (\r\n select\r\n fan.*,\r\n 1/(1+EXP(-(0.24602303+coef1+coef2+coef3+coef4+coef5+coef6+coef7+coef8+coef9+available_balance_coef+coef10+coef11+coef12+coef13+coef14))) as prediction_probability,\r\n case when prediction_probability < .5 then 0 else 1 end as prediction_class\r\n from apply_ceofficients ac\r\n inner join fill_na_values fan\r\n on ac.md5_surrogate_key_main = fan.md5_surrogate_key_main\r\n)\r\n\r\nselect * from logistic_prediction\r\n```\n@davesgonechina: I found a workaround if you want to try it. Don't use Jinja whitespace control. In other words, replace all occurrences of `{%-` with `{%` and all occurrences of `-%}` with `%}`.\r\n\r\nI'll keep looking to see if I can find a fix. SQLFluff has had some past bugs involving whitespace control. Basically, it makes SQLFluff's job more challenging, when it tries to \"map\" the input SQL (before running Jinja) to the output file (after running Jinja).\nIn the file `src/sqlfluff/core/templaters/slicers/tracer.py`, I thought that the recently added function `_remove_block_whitespace_control` would eliminate any issues with whitespace control. It was added to fix _some_ issues like this. Perhaps this is a more complex situation?\r\n\r\nGenerally, avoiding whitespace control in the \"alternate\" template results in template output with more \"breadcrumbs\", making it easier for the tracer to deduce the execution path of the template. The issue we saw before (which may be happening here) is that the tracer loses track of the execution path and \"drops\" off the end of the template at some point. Should be fairly easy to find where (and why) this is happening. May be harder to fix. We shall see...", "created_at": "2022-03-11T21:52:54Z", "version": "0.10", "FAIL_TO_PASS": ["test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[select\\n"], "PASS_TO_PASS": ["test/core/templaters/jinja_test.py::test__templater_jinja[simple]", "test/core/templaters/jinja_test.py::test__templater_jinja[unboundlocal_bugfix]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_left_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_block]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[basic_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_right_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_data]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[strip_both_comment]", "test/core/templaters/jinja_test.py::test__templater_jinja_slices[union_all_loop1]", "test/core/templaters/jinja_test.py::test__templater_jinja_error_variable", "test/core/templaters/jinja_test.py::test__templater_jinja_error_syntax", "test/core/templaters/jinja_test.py::test__templater_jinja_error_catastrophic", "test/core/templaters/jinja_test.py::test__templater_jinja_error_macro_path_does_not_exist", "test/core/templaters/jinja_test.py::test__templater_jinja_lint_empty", "test/core/templaters/jinja_test.py::test__templater_full[jinja_a/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_b/jinja-False-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/dbt_builtins-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_c_dbt/var_default-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_e/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_f/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_g_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_h_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_i_raw/raw_tag_2-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_j_libraries/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_k_config_override_path_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/001-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_l_metas/002-False-True]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_m_libraries_module/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_full[jinja_n_nested_macros/jinja-True-False]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[foo", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_template[{%", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[-None-result0]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[foo-None-result1]", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{{", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[SELECT\\n", "test/core/templaters/jinja_test.py::test__templater_jinja_slice_file[{%-"], "environment_setup_commit": "3d52e8270d82aeccf4c516d059a80a6947919aea"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-3662", "base_commit": "f9a3fe8f639d279226f16bdc51326dfa5c142c3e", "patch": "diff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py\n--- a/src/sqlfluff/cli/commands.py\n+++ b/src/sqlfluff/cli/commands.py\n@@ -486,8 +486,12 @@ def dump_file_payload(filename: Optional[str], payload: str):\n \"-p\",\n \"--processes\",\n type=int,\n- default=1,\n- help=\"The number of parallel processes to run.\",\n+ default=None,\n+ help=(\n+ \"The number of parallel processes to run. Positive numbers work as \"\n+ \"expected. Zero and negative numbers will work as number_of_cpus - \"\n+ \"number. e.g -1 means all cpus except one. 0 means all cpus.\"\n+ ),\n )\n @click.option(\n \"--disable_progress_bar\",\n@@ -497,7 +501,6 @@ def dump_file_payload(filename: Optional[str], payload: str):\n @click.argument(\"paths\", nargs=-1, type=click.Path(allow_dash=True))\n def lint(\n paths: Tuple[str],\n- processes: int,\n format: str,\n write_output: Optional[str],\n annotation_level: str,\n@@ -505,6 +508,7 @@ def lint(\n disregard_sqlfluffignores: bool,\n logger: Optional[logging.Logger] = None,\n bench: bool = False,\n+ processes: Optional[int] = None,\n disable_progress_bar: Optional[bool] = False,\n extra_config_path: Optional[str] = None,\n ignore_local_config: bool = False,\n@@ -675,8 +679,12 @@ def do_fixes(lnt, result, formatter=None, **kwargs):\n \"-p\",\n \"--processes\",\n type=int,\n- default=1,\n- help=\"The number of parallel processes to run.\",\n+ default=None,\n+ help=(\n+ \"The number of parallel processes to run. Positive numbers work as \"\n+ \"expected. Zero and negative numbers will work as number_of_cpus - \"\n+ \"number. e.g -1 means all cpus except one. 0 means all cpus.\"\n+ ),\n )\n @click.option(\n \"--disable_progress_bar\",\n@@ -705,10 +713,10 @@ def do_fixes(lnt, result, formatter=None, **kwargs):\n def fix(\n force: bool,\n paths: Tuple[str],\n- processes: int,\n bench: bool = False,\n fixed_suffix: str = \"\",\n logger: Optional[logging.Logger] = None,\n+ processes: Optional[int] = None,\n disable_progress_bar: Optional[bool] = False,\n extra_config_path: Optional[str] = None,\n ignore_local_config: bool = False,\ndiff --git a/src/sqlfluff/cli/formatters.py b/src/sqlfluff/cli/formatters.py\n--- a/src/sqlfluff/cli/formatters.py\n+++ b/src/sqlfluff/cli/formatters.py\n@@ -181,6 +181,14 @@ def dispatch_compilation_header(self, templater, message):\n f\"=== [{self.colorize(templater, Color.lightgrey)}] {message}\"\n ) # pragma: no cover\n \n+ def dispatch_processing_header(self, processes: int) -> None:\n+ \"\"\"Dispatch the header displayed before linting.\"\"\"\n+ if self._verbosity > 0:\n+ self._dispatch( # pragma: no cover\n+ f\"{self.colorize('effective configured processes: ', Color.lightgrey)} \"\n+ f\"{processes}\"\n+ )\n+\n def dispatch_dialect_warning(self, dialect) -> None:\n \"\"\"Dispatch a warning for dialects.\"\"\"\n self._dispatch(self.format_dialect_warning(dialect)) # pragma: no cover\ndiff --git a/src/sqlfluff/core/linter/linter.py b/src/sqlfluff/core/linter/linter.py\n--- a/src/sqlfluff/core/linter/linter.py\n+++ b/src/sqlfluff/core/linter/linter.py\n@@ -1066,7 +1066,7 @@ def lint_path(\n fix: bool = False,\n ignore_non_existent_files: bool = False,\n ignore_files: bool = True,\n- processes: int = 1,\n+ processes: Optional[int] = None,\n ) -> LintedDir:\n \"\"\"Lint a path.\"\"\"\n linted_path = LintedDir(path)\n@@ -1080,16 +1080,22 @@ def lint_path(\n )\n )\n \n+ if processes is None:\n+ processes = self.config.get(\"processes\", default=1)\n+\n # to avoid circular import\n from sqlfluff.core.linter.runner import get_runner\n \n- runner = get_runner(\n+ runner, effective_processes = get_runner(\n self,\n self.config,\n processes=processes,\n allow_process_parallelism=self.allow_process_parallelism,\n )\n \n+ if self.formatter and effective_processes != 1:\n+ self.formatter.dispatch_processing_header(effective_processes)\n+\n # Show files progress bar only when there is more than one.\n files_count = len(fnames)\n progress_bar_files = tqdm(\n@@ -1124,7 +1130,7 @@ def lint_paths(\n fix: bool = False,\n ignore_non_existent_files: bool = False,\n ignore_files: bool = True,\n- processes: int = 1,\n+ processes: Optional[int] = None,\n ) -> LintingResult:\n \"\"\"Lint an iterable of paths.\"\"\"\n paths_count = len(paths)\ndiff --git a/src/sqlfluff/core/linter/runner.py b/src/sqlfluff/core/linter/runner.py\n--- a/src/sqlfluff/core/linter/runner.py\n+++ b/src/sqlfluff/core/linter/runner.py\n@@ -10,6 +10,7 @@\n import bdb\n import functools\n import logging\n+import multiprocessing\n import multiprocessing.dummy\n import signal\n import sys\n@@ -227,15 +228,29 @@ def get_runner(\n config: FluffConfig,\n processes: int,\n allow_process_parallelism: bool = True,\n-) -> BaseRunner:\n- \"\"\"Generate a runner instance based on parallel and system configuration.\"\"\"\n+) -> Tuple[BaseRunner, int]:\n+ \"\"\"Generate a runner instance based on parallel and system configuration.\n+\n+ The processes argument can be positive or negative.\n+ - If positive, the integer is interpreted as the number of processes.\n+ - If negative or zero, the integer is interpreted as number_of_cpus - processes.\n+\n+ e.g.\n+ -1 = all cpus but one.\n+ 0 = all cpus\n+ 1 = 1 cpu\n+\n+ \"\"\"\n+ if processes <= 0:\n+ processes = max(multiprocessing.cpu_count() + processes, 1)\n+\n if processes > 1:\n # Process parallelism isn't really supported during testing\n # so this flag allows us to fall back to a threaded runner\n # in those cases.\n if allow_process_parallelism:\n- return MultiProcessRunner(linter, config, processes=processes)\n+ return MultiProcessRunner(linter, config, processes=processes), processes\n else:\n- return MultiThreadRunner(linter, config, processes=processes)\n+ return MultiThreadRunner(linter, config, processes=processes), processes\n else:\n- return SequentialRunner(linter, config)\n+ return SequentialRunner(linter, config), processes\n", "test_patch": "diff --git a/test/core/linter_test.py b/test/core/linter_test.py\n--- a/test/core/linter_test.py\n+++ b/test/core/linter_test.py\n@@ -14,6 +14,7 @@\n from sqlfluff.cli.formatters import OutputStreamFormatter\n from sqlfluff.cli.outputstream import make_output_stream\n from sqlfluff.core.linter import LintingResult, NoQaDirective\n+from sqlfluff.core.linter.runner import get_runner\n import sqlfluff.core.linter as linter\n from sqlfluff.core.parser import GreedyUntil, Ref\n from sqlfluff.core.templaters import TemplatedFile\n@@ -289,6 +290,36 @@ def test_lint_path_parallel_wrapper_exception(patched_lint):\n result.reraise()\n \n \n+@pytest.mark.parametrize(\n+ \"mock_cpu,in_processes,exp_processes\",\n+ [\n+ # Make the mocked cpu count a really high value which is\n+ # unlikely to collide with the real value. We can then\n+ # test all the different combos.\n+ (512, 1, 1),\n+ (512, 0, 512),\n+ (512, -12, 500),\n+ (512, 5, 5),\n+ # Check that we can't go lower than 1 in a 1 cpu case\n+ (1, -1, 1),\n+ ],\n+)\n+@patch(\"multiprocessing.cpu_count\")\n+def test__linter__get_runner_processes(\n+ patched_cpu_count, mock_cpu, in_processes, exp_processes\n+):\n+ \"\"\"Test that get_runner handles processes correctly.\"\"\"\n+ # Make the mocked cpu count a really high value which is\n+ # unlikely to collide with the real value.\n+ patched_cpu_count.return_value = mock_cpu\n+ _, return_processes = get_runner(\n+ linter=Linter(),\n+ config=FluffConfig(overrides={\"dialect\": \"ansi\"}),\n+ processes=in_processes,\n+ )\n+ assert return_processes == exp_processes\n+\n+\n @patch(\"sqlfluff.core.linter.runner.linter_logger\")\n @patch(\"sqlfluff.core.linter.Linter.lint_rendered\")\n def test__linter__linting_unexpected_error_handled_gracefully(\n", "problem_statement": "Number of processes configurable in .sqlfluff\nBeing able to set the number of processes to run with in .sqlfluff might be useful to avoid having to pass it in the CLI every time.\n", "hints_text": "One thought on this: The same `.sqlfluff` file will sometimes be used on different machines (e.g. various development machines, CI server). We should allow the setting to be somewhat \"context sensitive\" if desired. Proposal:\r\n* Positive values indicate the number of processes to create\r\n* Zero or negative values are interpreted as `number_of_cpus - specified_number`. Thus, a value of `0` means \"use all processors\" and `-1` means \"use all processors except one\".\nIs there a standard way in python to detect the effective available cpus?\n@alanmcruickshank: Yes. Use [`multiprocessing.cpu_count()`](https://docs.python.org/3/library/multiprocessing.html#multiprocessing.cpu_count).\nI'm happy to pick this one up. This actually fits with a small reorg of how I think threads should be configured. I think it fits better if the thread argument is passed in to the `Linter` object on instantiation, rather than when calling `lint_paths`. @barrywhart - does that sit well with you? I realise that changes a little of the structure you originally envisaged.\nSounds good -- no concerns from me.", "created_at": "2022-07-25T09:10:25Z", "version": "1.2", "FAIL_TO_PASS": ["test/core/linter_test.py::test__linter__get_runner_processes[512-1-1]", "test/core/linter_test.py::test__linter__get_runner_processes[512-0-512]", "test/core/linter_test.py::test__linter__get_runner_processes[512--12-500]", "test/core/linter_test.py::test__linter__get_runner_processes[512-5-5]", "test/core/linter_test.py::test__linter__get_runner_processes[1--1-1]"], "PASS_TO_PASS": ["test/core/linter_test.py::test__linter__path_from_paths__dir", "test/core/linter_test.py::test__linter__path_from_paths__default", "test/core/linter_test.py::test__linter__path_from_paths__exts", "test/core/linter_test.py::test__linter__path_from_paths__file", "test/core/linter_test.py::test__linter__path_from_paths__not_exist", "test/core/linter_test.py::test__linter__path_from_paths__not_exist_ignore", "test/core/linter_test.py::test__linter__path_from_paths__explicit_ignore", "test/core/linter_test.py::test__linter__path_from_paths__sqlfluffignore_current_directory", "test/core/linter_test.py::test__linter__path_from_paths__dot", "test/core/linter_test.py::test__linter__path_from_paths__ignore[test/fixtures/linter/sqlfluffignore]", "test/core/linter_test.py::test__linter__path_from_paths__ignore[test/fixtures/linter/sqlfluffignore/]", "test/core/linter_test.py::test__linter__path_from_paths__ignore[test/fixtures/linter/sqlfluffignore/.]", "test/core/linter_test.py::test__linter__lint_string_vs_file[test/fixtures/linter/indentation_errors.sql]", "test/core/linter_test.py::test__linter__lint_string_vs_file[test/fixtures/linter/whitespace_errors.sql]", "test/core/linter_test.py::test__linter__get_violations_filter_rules[None-7]", "test/core/linter_test.py::test__linter__get_violations_filter_rules[L010-2]", "test/core/linter_test.py::test__linter__get_violations_filter_rules[rules2-2]", "test/core/linter_test.py::test__linter__linting_result__sum_dicts", "test/core/linter_test.py::test__linter__linting_result__combine_dicts", "test/core/linter_test.py::test__linter__linting_result_check_tuples_by_path[False-list]", "test/core/linter_test.py::test__linter__linting_result_check_tuples_by_path[True-dict]", "test/core/linter_test.py::test__linter__linting_result_get_violations[1]", "test/core/linter_test.py::test__linter__linting_result_get_violations[2]", "test/core/linter_test.py::test__linter__linting_parallel_thread[False]", "test/core/linter_test.py::test__linter__linting_parallel_thread[True]", "test/core/linter_test.py::test_lint_path_parallel_wrapper_exception", "test/core/linter_test.py::test__linter__linting_unexpected_error_handled_gracefully", "test/core/linter_test.py::test__linter__raises_malformed_noqa", "test/core/linter_test.py::test__linter__empty_file", "test/core/linter_test.py::test__linter__mask_templated_violations[True-check_tuples0]", "test/core/linter_test.py::test__linter__mask_templated_violations[False-check_tuples1]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8.sql-autodetect-False]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8-sig.sql-autodetect-False]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8.sql-utf-8-False]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8-sig.sql-utf-8-True]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8.sql-utf-8-sig-False]", "test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8-sig.sql-utf-8-sig-False]", "test/core/linter_test.py::test_parse_noqa[-None]", "test/core/linter_test.py::test_parse_noqa[noqa-expected1]", "test/core/linter_test.py::test_parse_noqa[noqa?-SQLParseError]", "test/core/linter_test.py::test_parse_noqa[noqa:-expected3]", "test/core/linter_test.py::test_parse_noqa[noqa:L001,L002-expected4]", "test/core/linter_test.py::test_parse_noqa[noqa:", "test/core/linter_test.py::test_parse_noqa[Inline", "test/core/linter_test.py::test_parse_noqa_no_dups", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_no_ignore]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_specific_line]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_different_specific_line]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_different_specific_rule]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_enable_this_range]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_disable_this_range]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_1_ignore_disable_specific_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_2_ignore_disable_specific_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_3_ignore_disable_specific_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_4_ignore_disable_specific_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_1_ignore_disable_all_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_2_ignore_disable_all_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_3_ignore_disable_all_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_4_ignore_disable_all_2_3]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[4_violations_two_types_disable_specific_enable_all]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[4_violations_two_types_disable_all_enable_specific]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violations_comment_inline_ignore]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[2_violations_comment_inline_ignore]", "test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violations_comment_inline_glob_ignore]", "test/core/linter_test.py::test_linter_noqa", "test/core/linter_test.py::test_linter_noqa_with_templating", "test/core/linter_test.py::test_linter_noqa_template_errors", "test/core/linter_test.py::test_linter_noqa_prs", "test/core/linter_test.py::test_linter_noqa_tmp", "test/core/linter_test.py::test_linter_noqa_disable", "test/core/linter_test.py::test_delayed_exception", "test/core/linter_test.py::test__attempt_to_change_templater_warning", "test/core/linter_test.py::test_safe_create_replace_file[utf8_create]", "test/core/linter_test.py::test_safe_create_replace_file[utf8_update]", "test/core/linter_test.py::test_safe_create_replace_file[utf8_special_char]", "test/core/linter_test.py::test_safe_create_replace_file[incorrect_encoding]", "test/core/linter_test.py::test_advanced_api_methods", "test/core/linter_test.py::test_normalise_newlines", "test/core/linter_test.py::test_require_match_parse_grammar"], "environment_setup_commit": "388dd01e05c7dcb880165c7241ed4027d9d0171e"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-4997", "base_commit": "50bbffd4672aa17af2651f40d533cf55048b7524", "patch": "diff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py\n--- a/src/sqlfluff/cli/commands.py\n+++ b/src/sqlfluff/cli/commands.py\n@@ -1356,7 +1356,7 @@ def render(\n fname = path\n \n # Get file specific config\n- file_config.process_raw_file_for_config(raw_sql)\n+ file_config.process_raw_file_for_config(raw_sql, fname)\n rendered = lnt.render_string(raw_sql, fname, file_config, \"utf8\")\n \n if rendered.templater_violations:\ndiff --git a/src/sqlfluff/core/config.py b/src/sqlfluff/core/config.py\n--- a/src/sqlfluff/core/config.py\n+++ b/src/sqlfluff/core/config.py\n@@ -34,6 +34,16 @@\n ConfigElemType = Tuple[Tuple[str, ...], Any]\n \n \n+ALLOWABLE_LAYOUT_CONFIG_KEYS = (\n+ \"spacing_before\",\n+ \"spacing_after\",\n+ \"spacing_within\",\n+ \"line_position\",\n+ \"align_within\",\n+ \"align_scope\",\n+)\n+\n+\n @dataclass\n class _RemovedConfig:\n old_path: Tuple[str, ...]\n@@ -501,13 +511,19 @@ def _incorporate_vals(ctx: dict, vals: List[ConfigElemType]) -> dict:\n def _validate_configs(\n configs: Iterable[ConfigElemType], file_path\n ) -> List[ConfigElemType]:\n- \"\"\"Validate config elements against removed list.\"\"\"\n+ \"\"\"Validate config elements.\n+\n+ We validate in two ways:\n+ 1. Are these config settings removed or deprecated.\n+ 2. Are these config elements in the layout section _valid_.\n+ \"\"\"\n config_map = {cfg.old_path: cfg for cfg in REMOVED_CONFIGS}\n # Materialise the configs into a list to we can iterate twice.\n new_configs = list(configs)\n defined_keys = {k for k, _ in new_configs}\n validated_configs = []\n for k, v in new_configs:\n+ # First validate against the removed option list.\n if k in config_map.keys():\n formatted_key = \":\".join(k)\n removed_option = config_map[k]\n@@ -549,12 +565,37 @@ def _validate_configs(\n else:\n # Raise an error.\n raise SQLFluffUserError(\n- f\"Config file {file_path} set an outdated config \"\n+ f\"Config file {file_path!r} set an outdated config \"\n f\"value {formatted_key}.\\n\\n{removed_option.warning}\\n\\n\"\n \"See https://docs.sqlfluff.com/en/stable/configuration.html\"\n \" for more details.\"\n )\n \n+ # Second validate any layout configs for validity.\n+ # NOTE: For now we don't check that the \"type\" is a valid one\n+ # to reference, or that the values are valid. For the values,\n+ # these are likely to be rejected by the layout routines at\n+ # runtime. The last risk area is validating that the type is\n+ # a valid one.\n+ if k and k[0] == \"layout\":\n+ # Check for:\n+ # - Key length\n+ # - Key values\n+ if (\n+ # Key length must be 4\n+ (len(k) != 4)\n+ # Second value must (currently) be \"type\"\n+ or (k[1] != \"type\")\n+ # Last key value must be one of the allowable options.\n+ or (k[3] not in ALLOWABLE_LAYOUT_CONFIG_KEYS)\n+ ):\n+ raise SQLFluffUserError(\n+ f\"Config file {file_path!r} set an invalid `layout` option \"\n+ f\"value {':'.join(k)}.\\n\"\n+ \"See https://docs.sqlfluff.com/en/stable/layout.html\"\n+ \"#configuring-layout for more details.\"\n+ )\n+\n validated_configs.append((k, v))\n return validated_configs\n \n@@ -1094,7 +1135,7 @@ def iter_vals(self, cfg: Optional[dict] = None) -> Iterable[tuple]:\n for idnt, key, val in self.iter_vals(cfg=cfg[k]):\n yield (idnt + 1, key, val)\n \n- def process_inline_config(self, config_line: str):\n+ def process_inline_config(self, config_line: str, fname: str):\n \"\"\"Process an inline config command and update self.\"\"\"\n # Strip preceding comment marks\n if config_line.startswith(\"--\"):\n@@ -1108,19 +1149,23 @@ def process_inline_config(self, config_line: str):\n config_line = config_line[9:].strip()\n # Divide on colons\n config_path = [elem.strip() for elem in config_line.split(\":\")]\n+ config_val = (tuple(config_path[:-1]), config_path[-1])\n+ # Validate the value\n+ ConfigLoader._validate_configs([config_val], fname)\n # Set the value\n- self.set_value(config_path[:-1], config_path[-1])\n+ self.set_value(*config_val)\n # If the config is for dialect, initialise the dialect\n if config_path[:-1] == [\"dialect\"]:\n self._initialise_dialect(config_path[-1])\n \n- def process_raw_file_for_config(self, raw_str: str):\n+ def process_raw_file_for_config(self, raw_str: str, fname: str):\n \"\"\"Process a full raw file for inline config and update self.\"\"\"\n # Scan the raw file for config commands.\n for raw_line in raw_str.splitlines():\n- if raw_line.startswith(\"-- sqlfluff\"):\n+ # With or without a space.\n+ if raw_line.startswith((\"-- sqlfluff\", \"--sqlfluff\")):\n # Found a in-file config command\n- self.process_inline_config(raw_line)\n+ self.process_inline_config(raw_line, fname)\n \n \n class ProgressBarConfiguration:\ndiff --git a/src/sqlfluff/core/linter/linter.py b/src/sqlfluff/core/linter/linter.py\n--- a/src/sqlfluff/core/linter/linter.py\n+++ b/src/sqlfluff/core/linter/linter.py\n@@ -141,7 +141,7 @@ def load_raw_file_and_config(\n with open(fname, encoding=encoding, errors=\"backslashreplace\") as target_file:\n raw_file = target_file.read()\n # Scan the raw file for config commands.\n- file_config.process_raw_file_for_config(raw_file)\n+ file_config.process_raw_file_for_config(raw_file, fname)\n # Return the raw file and config\n return raw_file, file_config, encoding\n \n@@ -897,7 +897,7 @@ def parse_string(\n config = config or self.config\n \n # Scan the raw file for config commands.\n- config.process_raw_file_for_config(in_str)\n+ config.process_raw_file_for_config(in_str, fname)\n rendered = self.render_string(in_str, fname, config, encoding)\n violations += rendered.templater_violations\n \n", "test_patch": "diff --git a/test/core/config_test.py b/test/core/config_test.py\n--- a/test/core/config_test.py\n+++ b/test/core/config_test.py\n@@ -459,6 +459,50 @@ def test__config__validate_configs_indirect():\n )\n \n \n+@pytest.mark.parametrize(\n+ \"raw_sql\",\n+ [\n+ (\n+ # \"types\" not \"type\"\n+ \"-- sqlfluff:layout:types:comma:line_position:leading\\n\"\n+ \"SELECT 1\"\n+ ),\n+ (\n+ # Unsupported layout config length\n+ \"-- sqlfluff:layout:foo\\n\"\n+ \"SELECT 1\"\n+ ),\n+ (\n+ # Unsupported layout config length\n+ \"-- sqlfluff:layout:type:comma:bar\\n\"\n+ \"SELECT 1\"\n+ ),\n+ (\n+ # Unsupported layout config key (\"foo\")\n+ \"-- sqlfluff:layout:type:comma:foo:bar\\n\"\n+ \"SELECT 1\"\n+ ),\n+ (\n+ # Unsupported layout config key (\"foo\") [no space]\n+ \"--sqlfluff:layout:type:comma:foo:bar\\n\"\n+ \"SELECT 1\"\n+ ),\n+ ],\n+)\n+def test__config__validate_configs_inline_layout(raw_sql):\n+ \"\"\"Test _validate_configs method of FluffConfig when used on a file.\n+\n+ This test covers both the validation of inline config\n+ directives but also the validation of layout configs.\n+ \"\"\"\n+ # Instantiate config object.\n+ cfg = FluffConfig(configs={\"core\": {\"dialect\": \"ansi\"}})\n+\n+ # Try to process an invalid inline config. Make sure we get an error.\n+ with pytest.raises(SQLFluffUserError):\n+ cfg.process_raw_file_for_config(raw_sql, \"test.sql\")\n+\n+\n def test__config__validate_configs_precedence_same_file():\n \"\"\"Test _validate_configs method of FluffConfig where there's a conflict.\"\"\"\n # Check with a known conflicted value\n@@ -528,19 +572,19 @@ def test__process_inline_config():\n cfg = FluffConfig(config_b)\n assert cfg.get(\"rules\") == \"LT03\"\n \n- cfg.process_inline_config(\"-- sqlfluff:rules:LT02\")\n+ cfg.process_inline_config(\"-- sqlfluff:rules:LT02\", \"test.sql\")\n assert cfg.get(\"rules\") == \"LT02\"\n \n assert cfg.get(\"tab_space_size\", section=\"indentation\") == 4\n- cfg.process_inline_config(\"-- sqlfluff:indentation:tab_space_size:20\")\n+ cfg.process_inline_config(\"-- sqlfluff:indentation:tab_space_size:20\", \"test.sql\")\n assert cfg.get(\"tab_space_size\", section=\"indentation\") == 20\n \n assert cfg.get(\"dialect\") == \"ansi\"\n assert cfg.get(\"dialect_obj\").name == \"ansi\"\n- cfg.process_inline_config(\"-- sqlfluff:dialect:postgres\")\n+ cfg.process_inline_config(\"-- sqlfluff:dialect:postgres\", \"test.sql\")\n assert cfg.get(\"dialect\") == \"postgres\"\n assert cfg.get(\"dialect_obj\").name == \"postgres\"\n \n assert cfg.get(\"rulez\") is None\n- cfg.process_inline_config(\"-- sqlfluff:rulez:LT06\")\n+ cfg.process_inline_config(\"-- sqlfluff:rulez:LT06\", \"test.sql\")\n assert cfg.get(\"rulez\") == \"LT06\"\n", "problem_statement": "Validate layout configurations on load\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### Description\n\nAs raised in this comment: https://github.com/sqlfluff/sqlfluff/pull/4558#discussion_r1142745101\r\n\r\nAt the moment, the layout configs are being validated _on use_ which is potentially flaky and convoluted. Better would be to validate configs _on load_.\n\n### Use case\n\n_No response_\n\n### Dialect\n\nall\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [X] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "", "created_at": "2023-07-24T14:00:41Z", "version": "2.0", "FAIL_TO_PASS": ["test/core/config_test.py::test__config__validate_configs_inline_layout[--", "test/core/config_test.py::test__config__validate_configs_inline_layout[--sqlfluff:layout:type:comma:foo:bar\\nSELECT", "test/core/config_test.py::test__process_inline_config"], "PASS_TO_PASS": ["test/core/config_test.py::test__config__nested_combine", "test/core/config_test.py::test__config__dict_diff", "test/core/config_test.py::test__config__load_file_dir", "test/core/config_test.py::test__config__load_file_f", "test/core/config_test.py::test__config__load_nested", "test/core/config_test.py::test__config__iter_config_elems_from_dict", "test/core/config_test.py::test__config__load_toml", "test/core/config_test.py::test__config__load_placeholder_cfg", "test/core/config_test.py::test__config__iter_config_paths_right_order", "test/core/config_test.py::test__config__find_sqlfluffignore_in_same_directory", "test/core/config_test.py::test__config__nested_config_tests", "test/core/config_test.py::test__config__load_user_appdir_config", "test/core/config_test.py::test__config__split_comma_separated_string[AL01,LT08,AL07-expected0]", "test/core/config_test.py::test__config__split_comma_separated_string[\\nAL01,\\nLT08,\\nAL07,-expected1]", "test/core/config_test.py::test__config__split_comma_separated_string[raw_str2-expected2]", "test/core/config_test.py::test__config__split_comma_separated_string_correct_type", "test/core/config_test.py::test__config__templater_selection", "test/core/config_test.py::test__config__glob_exclude_config_tests", "test/core/config_test.py::test__config__glob_include_config_tests", "test/core/config_test.py::test__config__rules_set_to_none", "test/core/config_test.py::test__config__rules_group_with_exclude", "test/core/config_test.py::test__config__get_section", "test/core/config_test.py::test__config__get", "test/core/config_test.py::test__config__from_kwargs", "test/core/config_test.py::test__config_missing_dialect", "test/core/config_test.py::test__config__validate_configs_direct", "test/core/config_test.py::test__config__validate_configs_indirect", "test/core/config_test.py::test__config__validate_configs_precedence_same_file", "test/core/config_test.py::test__config__toml_list_config", "test/core/config_test.py::test__config__warn_unknown_rule"], "environment_setup_commit": "3629c3e702939c07264cc5ea903566ddc9ea2bb0"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-2998", "base_commit": "47c8bb29104761474e455ef2e6fdaa7a8cc20a56", "patch": "diff --git a/src/sqlfluff/rules/L027.py b/src/sqlfluff/rules/L027.py\n--- a/src/sqlfluff/rules/L027.py\n+++ b/src/sqlfluff/rules/L027.py\n@@ -73,4 +73,21 @@ def _lint_references_and_aliases(\n )\n )\n \n+ all_table_aliases = [t.ref_str for t in table_aliases] + standalone_aliases\n+\n+ # For qualified references, we want to check that the alias is actually\n+ # valid\n+ if (\n+ this_ref_type == \"qualified\"\n+ and list(r.iter_raw_references())[0].part not in all_table_aliases\n+ ):\n+ violation_buff.append(\n+ LintResult(\n+ anchor=r,\n+ description=f\"Qualified reference {r.raw!r} not found in \"\n+ f\"available tables/view aliases {all_table_aliases} in select \"\n+ \"with more than one referenced table/view.\",\n+ )\n+ )\n+\n return violation_buff or None\n", "test_patch": "diff --git a/test/fixtures/rules/std_rule_cases/L027.yml b/test/fixtures/rules/std_rule_cases/L027.yml\n--- a/test/fixtures/rules/std_rule_cases/L027.yml\n+++ b/test/fixtures/rules/std_rule_cases/L027.yml\n@@ -220,3 +220,40 @@ test_pass_rowtype_with_join:\n configs:\n core:\n dialect: hive\n+\n+test_fail_column_name_not_found_in_table_aliases_bigquery:\n+ # qualified reference should actually exists in table aliases\n+ fail_str: |\n+ SELECT\n+ a.bar,\n+ b.foo,\n+ this_is.some_struct.id\n+ FROM\n+ a LEFT JOIN b ON TRUE\n+ configs:\n+ core:\n+ dialect: bigquery\n+\n+test_pass_column_name_is_a_struct_bigquery:\n+ # check structs work as expected\n+ pass_str: |\n+ SELECT\n+ a.bar,\n+ b.this_is.some_struct.id\n+ FROM\n+ a LEFT JOIN b ON TRUE\n+ configs:\n+ core:\n+ dialect: bigquery\n+\n+test_pass_column_name_from_unnest_bigquery:\n+ # Check that we allow an table alias come from UNNEST statement\n+ pass_str: |\n+ SELECT\n+ a.bar,\n+ e.foo\n+ FROM\n+ a LEFT JOIN UNEST(a.events) AS e\n+ configs:\n+ core:\n+ dialect: bigquery\ndiff --git a/test/rules/std_test.py b/test/rules/std_test.py\n--- a/test/rules/std_test.py\n+++ b/test/rules/std_test.py\n@@ -68,7 +68,7 @@\n ),\n (\"L016\", \"block_comment_errors_2.sql\", [(1, 85), (2, 86)]),\n # Column references\n- (\"L027\", \"column_references.sql\", [(1, 8)]),\n+ (\"L027\", \"column_references.sql\", [(1, 8), (1, 11)]),\n (\"L027\", \"column_references_bare_function.sql\", []),\n (\"L026\", \"column_references.sql\", [(1, 11)]),\n (\"L025\", \"column_references.sql\", [(2, 11)]),\n", "problem_statement": "BigQuery: Accessing `STRUCT` elements evades triggering L027\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nAccessing unreferenced `STRUCT` elements using BigQuery dot notation in a multi table query does not trigger L027.\n\n### Expected Behaviour\n\nL027 gets triggered.\n\n### Observed Behaviour\n\nL027 does not get triggered.\n\n### How to reproduce\n\n```sql\r\nSELECT\r\n t1.col1,\r\n t2.col2,\r\n events.id\r\nFROM t_table1 AS t1\r\nLEFT JOIN t_table2 AS t2\r\n ON TRUE\r\n```\n\n### Dialect\n\nBigQUery\n\n### Version\n\n`0.11.2` using online.sqlfluff.com\n\n### Configuration\n\nN/A\n\n### Are you willing to work on and submit a PR to address the issue?\n\n- [ ] Yes I am willing to submit a PR!\n\n### Code of Conduct\n\n- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)\n\n", "hints_text": "This is tricky.\r\n\r\nBasicaly L026 works to make sure qualified columns only use tables in the from clause. This doesn\u2019t really work for `STRUCT`s as impossible to know if it\u2019s a qualified column or a `STRUCT`, so is off by default for languages that support them - like BigQuery.\r\n\r\nL027 works to make sure columns are qualified for multi-table joins (i.e. have at least one dot). But it doesn\u2019t check the qualifiers are valid - that\u2019s L026\u2019s job, which as I say is off by default for BigQuery.", "created_at": "2022-04-04T20:29:42Z", "version": "0.11", "FAIL_TO_PASS": ["test/rules/std_test.py::test__rules__std_file[L027-column_references.sql-violations16]"], "PASS_TO_PASS": ["test/rules/std_test.py::test__rules__std_file[L001-indentation_errors.sql-violations0]", "test/rules/std_test.py::test__rules__std_file[L002-indentation_errors.sql-violations1]", "test/rules/std_test.py::test__rules__std_file[L003-indentation_errors.sql-violations2]", "test/rules/std_test.py::test__rules__std_file[L004-indentation_errors.sql-violations3]", "test/rules/std_test.py::test__rules__std_file[L005-whitespace_errors.sql-violations4]", "test/rules/std_test.py::test__rules__std_file[L019-whitespace_errors.sql-violations5]", "test/rules/std_test.py::test__rules__std_file[L008-whitespace_errors.sql-violations6]", "test/rules/std_test.py::test__rules__std_file[L006-operator_errors.sql-violations7]", "test/rules/std_test.py::test__rules__std_file[L039-operator_errors.sql-violations8]", "test/rules/std_test.py::test__rules__std_file[L007-operator_errors.sql-violations9]", "test/rules/std_test.py::test__rules__std_file[L006-operator_errors_negative.sql-violations10]", "test/rules/std_test.py::test__rules__std_file[L039-operator_errors_negative.sql-violations11]", "test/rules/std_test.py::test__rules__std_file[L003-indentation_error_hard.sql-violations12]", "test/rules/std_test.py::test__rules__std_file[L003-indentation_error_contained.sql-violations13]", "test/rules/std_test.py::test__rules__std_file[L016-block_comment_errors.sql-violations14]", "test/rules/std_test.py::test__rules__std_file[L016-block_comment_errors_2.sql-violations15]", "test/rules/std_test.py::test__rules__std_file[L027-column_references_bare_function.sql-violations17]", "test/rules/std_test.py::test__rules__std_file[L026-column_references.sql-violations18]", "test/rules/std_test.py::test__rules__std_file[L025-column_references.sql-violations19]", "test/rules/std_test.py::test__rules__std_file[L021-select_distinct_group_by.sql-violations20]", "test/rules/std_test.py::test__rules__std_file[L006-operator_errors_ignore.sql-violations21]", "test/rules/std_test.py::test__rules__std_file[L031-aliases_in_join_error.sql-violations22]", "test/rules/std_test.py::test__rules__std_file[L046-heavy_templating.sql-violations23]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict0]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict1]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict2]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict3]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict4]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict5]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict6]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict7]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict8]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict9]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict10]", "test/rules/std_test.py::test_improper_configs_are_rejected[rule_config_dict11]"], "environment_setup_commit": "2bdeb9354d33e3fb4dfd6782e1e1921939ecb55a"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-1733", "base_commit": "a1579a16b1d8913d9d7c7d12add374a290bcc78c", "patch": "diff --git a/src/sqlfluff/rules/L039.py b/src/sqlfluff/rules/L039.py\n--- a/src/sqlfluff/rules/L039.py\n+++ b/src/sqlfluff/rules/L039.py\n@@ -44,7 +44,9 @@ def _eval(self, context: RuleContext) -> Optional[List[LintResult]]:\n # This is to avoid indents\n if not prev_newline:\n prev_whitespace = seg\n- prev_newline = False\n+ # We won't set prev_newline to False, just for whitespace\n+ # in case there's multiple indents, inserted by other rule\n+ # fixes (see #1713)\n elif seg.is_type(\"comment\"):\n prev_newline = False\n prev_whitespace = None\n", "test_patch": "diff --git a/test/rules/std_L003_L036_L039_combo_test.py b/test/rules/std_L003_L036_L039_combo_test.py\nnew file mode 100644\n--- /dev/null\n+++ b/test/rules/std_L003_L036_L039_combo_test.py\n@@ -0,0 +1,36 @@\n+\"\"\"Tests issue #1373 doesn't reoccur.\n+\n+The combination of L003 (incorrect indentation), L036 (select targets),\n+and L039 (unnecessary white space) can result in incorrect indentation.\n+\"\"\"\n+\n+import sqlfluff\n+\n+\n+def test__rules__std_L003_L036_L039():\n+ \"\"\"Verify that double indents don't flag L039.\"\"\"\n+ sql = \"\"\"\n+ WITH example AS (\n+ SELECT my_id,\n+ other_thing,\n+ one_more\n+ FROM\n+ my_table\n+ )\n+\n+ SELECT *\n+ FROM example\\n\"\"\"\n+ fixed_sql = \"\"\"\n+ WITH example AS (\n+ SELECT\n+ my_id,\n+ other_thing,\n+ one_more\n+ FROM\n+ my_table\n+ )\n+\n+ SELECT *\n+ FROM example\\n\"\"\"\n+ result = sqlfluff.fix(sql)\n+ assert result == fixed_sql\ndiff --git a/test/rules/std_L016_L36_combo.py b/test/rules/std_L016_L36_combo_test.py\nsimilarity index 100%\nrename from test/rules/std_L016_L36_combo.py\nrename to test/rules/std_L016_L36_combo_test.py\n", "problem_statement": "Extra space when first field moved to new line in a WITH statement\nNote, the query below uses a `WITH` statement. If I just try to fix the SQL within the CTE, this works fine.\r\n\r\nGiven the following SQL:\r\n\r\n```sql\r\nWITH example AS (\r\n SELECT my_id,\r\n other_thing,\r\n one_more\r\n FROM\r\n my_table\r\n)\r\n\r\nSELECT *\r\nFROM example\r\n```\r\n\r\n## Expected Behaviour\r\n\r\nafter running `sqlfluff fix` I'd expect (`my_id` gets moved down and indented properly):\r\n\r\n```sql\r\nWITH example AS (\r\n SELECT\r\n my_id,\r\n other_thing,\r\n one_more\r\n FROM\r\n my_table\r\n)\r\n\r\nSELECT *\r\nFROM example\r\n```\r\n\r\n## Observed Behaviour\r\n\r\nafter running `sqlfluff fix` we get (notice that `my_id` is indented one extra space)\r\n\r\n```sql\r\nWITH example AS (\r\n SELECT\r\n my_id,\r\n other_thing,\r\n one_more\r\n FROM\r\n my_table\r\n)\r\n\r\nSELECT *\r\nFROM example\r\n```\r\n\r\n## Steps to Reproduce\r\n\r\nNoted above. Create a file with the initial SQL and fun `sqfluff fix` on it.\r\n\r\n## Dialect\r\n\r\nRunning with default config.\r\n\r\n## Version\r\nInclude the output of `sqlfluff --version` along with your Python version\r\n\r\nsqlfluff, version 0.7.0\r\nPython 3.7.5\r\n\r\n## Configuration\r\n\r\nDefault config.\r\n\n", "hints_text": "Does running `sqlfluff fix` again correct the SQL?\n@tunetheweb yes, yes it does. Is that something that the user is supposed to do (run it multiple times) or is this indeed a bug?\nIdeally not, but there are some circumstances where it\u2019s understandable that would happen. This however seems an easy enough example where it should not happen.\nThis appears to be a combination of rules L036, L003, and L039 not playing nicely together.\r\n\r\nThe original error is rule L036 and it produces this:\r\n\r\n```sql\r\nWITH example AS (\r\n SELECT\r\nmy_id,\r\n other_thing,\r\n one_more\r\n FROM\r\n my_table\r\n)\r\n\r\nSELECT *\r\nFROM example\r\n```\r\n\r\nThat is, it moves the `my_id` down to the newline but does not even try to fix the indentation.\r\n\r\nThen we have another run through and L003 spots the lack of indentation and fixes it by adding the first set of whitespace:\r\n\r\n```sql\r\nWITH example AS (\r\n SELECT\r\n my_id,\r\n other_thing,\r\n one_more\r\n FROM\r\n my_table\r\n)\r\n\r\nSELECT *\r\nFROM example\r\n```\r\n\r\nThen we have another run through and L003 spots that there still isn't enough indentation and fixes it by adding the second set of whitespace:\r\n\r\n```sql\r\nWITH example AS (\r\n SELECT\r\n my_id,\r\n other_thing,\r\n one_more\r\n FROM\r\n my_table\r\n)\r\n\r\nSELECT *\r\nFROM example\r\n```\r\n\r\nAt this point we're all good.\r\n\r\nHowever then L039 has a look. It never expects two sets of whitespace following a new line and is specifically coded to only assume one set of spaces (which it normally would be if the other rules hadn't interfered as it would be parsed as one big space), so it think's the second set is too much indentation, so it replaces it with a single space.\r\n\r\nThen another run and L003 and the whitespace back in so we end up with two indents, and a single space.\r\n\r\nLuckily the fix is easier than that explanation. PR coming up...\r\n\r\n", "created_at": "2021-10-22T18:23:33Z", "version": "0.6", "FAIL_TO_PASS": ["test/rules/std_L003_L036_L039_combo_test.py::test__rules__std_L003_L036_L039"], "PASS_TO_PASS": ["test/rules/std_L016_L36_combo_test.py::test__rules__std_L016_L036_long_line_lint", "test/rules/std_L016_L36_combo_test.py::test__rules__std_L016_L036_long_line_fix", "test/rules/std_L016_L36_combo_test.py::test__rules__std_L016_L036_long_line_fix2"], "environment_setup_commit": "67023b85c41d23d6c6d69812a41b207c4f8a9331"}, {"repo": "sqlfluff/sqlfluff", "instance_id": "sqlfluff__sqlfluff-3648", "base_commit": "e56fc6002dac0fb7eb446d58bd8aa7a839908535", "patch": "diff --git a/src/sqlfluff/core/templaters/slicers/tracer.py b/src/sqlfluff/core/templaters/slicers/tracer.py\n--- a/src/sqlfluff/core/templaters/slicers/tracer.py\n+++ b/src/sqlfluff/core/templaters/slicers/tracer.py\n@@ -206,7 +206,8 @@ def __init__(self, raw_str: str, env: Environment):\n \n # Internal bookkeeping\n self.slice_id: int = 0\n- self.inside_set_or_macro: bool = False # {% set %} or {% macro %}\n+ # {% set %} or {% macro %} or {% call %}\n+ self.inside_set_macro_or_call: bool = False\n self.inside_block = False # {% block %}\n self.stack: List[int] = []\n self.idx_raw: int = 0\n@@ -236,12 +237,18 @@ def slice_info_for_literal(self, length, prefix=\"\") -> RawSliceInfo:\n unique_alternate_id, alternate_code, inside_block=self.inside_block\n )\n \n- def update_inside_set_or_macro_or_block(\n- self, block_type: str, trimmed_parts: List[str]\n- ) -> None:\n- \"\"\"Based on block tag, update whether we're in a set/macro section.\"\"\"\n+ def update_inside_set_call_macro_or_block(\n+ self,\n+ block_type: str,\n+ trimmed_parts: List[str],\n+ m_open: Optional[regex.Match],\n+ m_close: Optional[regex.Match],\n+ tag_contents: List[str],\n+ ) -> Optional[RawSliceInfo]:\n+ \"\"\"Based on block tag, update whether in a set/call/macro/block section.\"\"\"\n if block_type == \"block_start\" and trimmed_parts[0] in (\n \"block\",\n+ \"call\",\n \"macro\",\n \"set\",\n ):\n@@ -274,16 +281,22 @@ def update_inside_set_or_macro_or_block(\n if trimmed_parts[0] == \"block\":\n self.inside_block = True\n else:\n- self.inside_set_or_macro = True\n+ result = None\n+ if trimmed_parts[0] == \"call\":\n+ assert m_open and m_close\n+ result = self.track_call(m_open, m_close, tag_contents)\n+ self.inside_set_macro_or_call = True\n+ return result\n else:\n raise # pragma: no cover\n elif block_type == \"block_end\":\n- if trimmed_parts[0] in (\"endmacro\", \"endset\"):\n- # Exiting a set or macro.\n- self.inside_set_or_macro = False\n+ if trimmed_parts[0] in (\"endcall\", \"endmacro\", \"endset\"):\n+ # Exiting a set or macro or block.\n+ self.inside_set_macro_or_call = False\n elif trimmed_parts[0] == \"endblock\":\n # Exiting a {% block %} block.\n self.inside_block = False\n+ return None\n \n def make_raw_slice_info(\n self,\n@@ -292,7 +305,7 @@ def make_raw_slice_info(\n inside_block: bool = False,\n ) -> RawSliceInfo:\n \"\"\"Create RawSliceInfo as given, or \"empty\" if in set/macro block.\"\"\"\n- if not self.inside_set_or_macro:\n+ if not self.inside_set_macro_or_call:\n return RawSliceInfo(unique_alternate_id, alternate_code, [], inside_block)\n else:\n return RawSliceInfo(None, None, [], False)\n@@ -355,6 +368,8 @@ def analyze(self, make_template: Callable[[str], Template]) -> JinjaTracer:\n # raw_end and raw_begin behave a little differently in\n # that the whole tag shows up in one go rather than getting\n # parts of the tag at a time.\n+ m_open = None\n+ m_close = None\n if elem_type.endswith(\"_end\") or elem_type == \"raw_begin\":\n block_type = self.block_types[elem_type]\n block_subtype = None\n@@ -376,7 +391,11 @@ def analyze(self, make_template: Callable[[str], Template]) -> JinjaTracer:\n raw_slice_info = self.track_templated(\n m_open, m_close, tag_contents\n )\n- self.update_inside_set_or_macro_or_block(block_type, tag_contents)\n+ raw_slice_info_temp = self.update_inside_set_call_macro_or_block(\n+ block_type, tag_contents, m_open, m_close, tag_contents\n+ )\n+ if raw_slice_info_temp:\n+ raw_slice_info = raw_slice_info_temp\n m_strip_right = regex.search(\n r\"\\s+$\", raw, regex.MULTILINE | regex.DOTALL\n )\n@@ -428,6 +447,7 @@ def analyze(self, make_template: Callable[[str], Template]) -> JinjaTracer:\n slice_idx = len(self.raw_sliced) - 1\n self.idx_raw += len(str_buff)\n if block_type.startswith(\"block\"):\n+ self.track_block_start(block_type, tag_contents[0])\n self.track_block_end(block_type, tag_contents[0])\n self.update_next_slice_indices(\n slice_idx, block_type, tag_contents[0]\n@@ -457,6 +477,21 @@ def track_templated(\n )\n return self.make_raw_slice_info(unique_alternate_id, alternate_code)\n \n+ def track_call(\n+ self, m_open: regex.Match, m_close: regex.Match, tag_contents: List[str]\n+ ):\n+ \"\"\"Set up tracking for \"{% call ... %}\".\"\"\"\n+ unique_alternate_id = self.next_slice_id()\n+ open_ = m_open.group(1)\n+ close_ = m_close.group(1)\n+ # Here, we still need to evaluate the original tag contents, e.g. in\n+ # case it has intentional side effects, but also return a slice ID\n+ # for tracking.\n+ alternate_code = (\n+ f\"\\0{unique_alternate_id} {open_} \" f\"{''.join(tag_contents)} {close_}\"\n+ )\n+ return self.make_raw_slice_info(unique_alternate_id, alternate_code)\n+\n def track_literal(self, raw: str, block_idx: int) -> None:\n \"\"\"Set up tracking for a Jinja literal.\"\"\"\n self.raw_sliced.append(\n@@ -517,6 +552,20 @@ def extract_tag_contents(\n trimmed_parts = trimmed_content.split()\n return trimmed_parts\n \n+ def track_block_start(self, block_type: str, tag_name: str) -> None:\n+ \"\"\"On starting a 'call' block, set slice_type to \"templated\".\"\"\"\n+ if block_type == \"block_start\" and tag_name == \"call\":\n+ # Replace RawSliceInfo for this slice with one that has block_type\n+ # \"templated\".\n+ old_raw_file_slice = self.raw_sliced[-1]\n+ self.raw_sliced[-1] = old_raw_file_slice._replace(slice_type=\"templated\")\n+\n+ # Move existing raw_slice_info entry since it's keyed by RawFileSlice.\n+ self.raw_slice_info[self.raw_sliced[-1]] = self.raw_slice_info[\n+ old_raw_file_slice\n+ ]\n+ del self.raw_slice_info[old_raw_file_slice]\n+\n def track_block_end(self, block_type: str, tag_name: str) -> None:\n \"\"\"On ending a 'for' or 'if' block, set up tracking.\"\"\"\n if block_type == \"block_end\" and tag_name in (\n@@ -553,7 +602,7 @@ def update_next_slice_indices(\n \"endfor\",\n \"endif\",\n ):\n- if not self.inside_set_or_macro:\n+ if not self.inside_set_macro_or_call:\n # Record potential forward jump over this block.\n self.raw_slice_info[\n self.raw_sliced[self.stack[-1]]\n", "test_patch": "diff --git a/test/core/templaters/jinja_test.py b/test/core/templaters/jinja_test.py\n--- a/test/core/templaters/jinja_test.py\n+++ b/test/core/templaters/jinja_test.py\n@@ -697,6 +697,14 @@ def test__templater_jinja_slice_template(test, result):\n ] == result\n \n \n+def _statement(*args, **kwargs):\n+ return \"_statement\"\n+\n+\n+def _load_result(*args, **kwargs):\n+ return \"_load_result\"\n+\n+\n @pytest.mark.parametrize(\n \"raw_file,override_context,result\",\n [\n@@ -1118,6 +1126,32 @@ def test__templater_jinja_slice_template(test, result):\n (\"literal\", slice(131, 132, None), slice(88, 89, None)),\n ],\n ),\n+ (\n+ \"\"\"{{ statement('variables', fetch_result=true) }}\n+\"\"\",\n+ dict(\n+ statement=_statement,\n+ load_result=_load_result,\n+ ),\n+ [\n+ (\"templated\", slice(0, 47, None), slice(0, 10, None)),\n+ (\"literal\", slice(47, 48, None), slice(10, 11, None)),\n+ ],\n+ ),\n+ (\n+ \"\"\"{% call statement('variables', fetch_result=true) %}select 1 as test{% endcall %}\n+\"\"\",\n+ dict(\n+ statement=_statement,\n+ load_result=_load_result,\n+ ),\n+ [\n+ (\"templated\", slice(0, 52, None), slice(0, 10, None)),\n+ (\"literal\", slice(52, 68, None), slice(10, 10, None)),\n+ (\"block_end\", slice(68, 81, None), slice(10, 10, None)),\n+ (\"literal\", slice(81, 82, None), slice(10, 11, None)),\n+ ],\n+ ),\n ],\n )\n def test__templater_jinja_slice_file(raw_file, override_context, result, caplog):\n", "problem_statement": "dbt & JinjaTracer results in passing invalid query to database (was: DBT Call statement() block causes invalid query generated)\n### Search before asking\n\n- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.\n\n\n### What Happened\n\nWhen using the call statement() to run a query during compile time, the query generated is garbled causing the following sql error:\r\n```\r\n{% call statement('variables', fetch_result=true) %}\r\n\r\nselect 1 as test;\r\n\r\n{% endcall %}\r\n\r\n{% set test = load_result('variables')['table'].columns.TEST.values()[0] %}\r\n```\r\n\r\nThis results in the following error:\r\n\r\ndbt.exceptions.DatabaseException: Database Error\r\n 001003 (42000): SQL compilation error:\r\n syntax error line 1 at position 0 unexpected '0'.\r\n\r\nThe query ran looks like this when looking at the query runner history in snowflake:\r\n\r\n```\r\n\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a_0\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a_8\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a\u263a_0\r\n```\r\n\r\nWhereas it should show:\r\n```\r\nselect 1 as test;\r\n```\n\n### Expected Behaviour\n\nExpected that the query runs properly.\n\n### Observed Behaviour\n\n```\r\n=== [dbt templater] Compiling dbt project...\r\n=== [dbt templater] Project Compiled.\r\nTraceback (most recent call last):\r\n File \"/usr/local/lib/python3.9/site-packages/dbt/adapters/snowflake/connections.py\", line 219, in exception_handler\r\n yield\r\n File \"/usr/local/lib/python3.9/site-packages/dbt/adapters/sql/connections.py\", line 70, in add_query\r\n cursor.execute(sql, bindings)\r\n File \"/usr/local/lib/python3.9/site-packages/snowflake/connector/cursor.py\", line 794, in execute\r\n Error.errorhandler_wrapper(self.connection, self, error_class, errvalue)\r\n File \"/usr/local/lib/python3.9/site-packages/snowflake/connector/errors.py\", line 273, in errorhandler_wrapper\r\n handed_over = Error.hand_to_other_handler(\r\n File \"/usr/local/lib/python3.9/site-packages/snowflake/connector/errors.py\", line 328, in hand_to_other_handler\r\n cursor.errorhandler(connection, cursor, error_class, error_value)\r\n File \"/usr/local/lib/python3.9/site-packages/snowflake/connector/errors.py\", line 207, in default_errorhandler\r\n raise error_class(\r\nsnowflake.connector.errors.ProgrammingError: 001003 (42000): SQL compilation error:\r\nsyntax error line 1 at position 0 unexpected '0'.\r\n\r\nDuring handling of the above exception, another exception occurred:\r\n\r\nTraceback (most recent call last):\r\n File \"/usr/local/bin/sqlfluff\", line 8, in \r\n sys.exit(cli())\r\n File \"/usr/local/lib/python3.9/site-packages/click/core.py\", line 1130, in __call__\r\n return self.main(*args, **kwargs)\r\n File \"/usr/local/lib/python3.9/site-packages/click/core.py\", line 1055, in main\r\n rv = self.invoke(ctx)\r\n File \"/usr/local/lib/python3.9/site-packages/click/core.py\", line 1657, in invoke\r\n return _process_result(sub_ctx.command.invoke(sub_ctx))\r\n File \"/usr/local/lib/python3.9/site-packages/click/core.py\", line 1404, in invoke\r\n return ctx.invoke(self.callback, **ctx.params)\r\n File \"/usr/local/lib/python3.9/site-packages/click/core.py\", line 760, in invoke\r\n return __callback(*args, **kwargs)\r\n File \"/usr/local/lib/python3.9/site-packages/sqlfluff/cli/commands.py\", line 1008, in parse\r\n parsed_strings = list(\r\n File \"/usr/local/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py\", line 1171, in parse_path\r\n yield self.parse_string(\r\n File \"/usr/local/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py\", line 835, in parse_string\r\n rendered = self.render_string(in_str, fname, config, encoding)\r\n File \"/usr/local/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py\", line 784, in render_string\r\n templated_file, templater_violations = self.templater.process(\r\n File \"/usr/local/lib/python3.9/site-packages/sqlfluff/core/templaters/base.py\", line 47, in _wrapped\r\n return func(self, in_str=in_str, fname=fname, config=config, **kwargs)\r\n File \"/usr/local/lib/python3.9/site-packages/sqlfluff_templater_dbt/templater.py\", line 331, in process\r\n processed_result = self._unsafe_process(fname_absolute_path, in_str, config)\r\n File \"/usr/local/lib/python3.9/site-packages/sqlfluff_templater_dbt/templater.py\", line 552, in _unsafe_process\r\n raw_sliced, sliced_file, templated_sql = self.slice_file(\r\n File \"/usr/local/lib/python3.9/site-packages/sqlfluff/core/templaters/jinja.py\", line 462, in slice_file\r\n trace = tracer.trace(append_to_templated=kwargs.pop(\"append_to_templated\", \"\"))\r\n File \"/usr/local/lib/python3.9/site-packages/sqlfluff/core/templaters/slicers/tracer.py\", line 77, in trace\r\n trace_template_output = trace_template.render()\r\n File \"/usr/local/lib/python3.9/site-packages/jinja2/environment.py\", line 1090, in render\r\n self.environment.handle_exception()\r\n File \"/usr/local/lib/python3.9/site-packages/jinja2/environment.py\", line 832, in handle_exception\r\n reraise(*rewrite_traceback_stack(source=source))\r\n File \"/usr/local/lib/python3.9/site-packages/jinja2/_compat.py\", line 28, in reraise\r\n raise value.with_traceback(tb)\r\n File \"