Source src/render/highlight.rs
1//! Lexical source fragments using rustdoc's CSS classes, never raw HTML.2// Classification follows rustdoc html/highlight.rs; see HIGHLIGHTING.md for3// source references, selected MIT licensing and compiler-context limitations.4use ra_ap_rustc_lexer::{FrontmatterAllowed, LiteralKind, TokenKind, strip_shebang, tokenize};56pub(super) struct Fragment {7 pub text: String,8 pub class: Option<&'static str>,9}1011struct Token<'a> {12 kind: TokenKind,13 text: &'a str,14}15fn trivia(kind: TokenKind) -> bool {16 matches!(17 kind,18 TokenKind::Whitespace | TokenKind::LineComment { .. } | TokenKind::BlockComment { .. }19 )20}21fn keyword(text: &str, next: Option<&Token<'_>>) -> bool {22 matches!(23 text,24 "_" | "as"25 | "async"26 | "await"27 | "break"28 | "const"29 | "continue"30 | "crate"31 | "dyn"32 | "else"33 | "enum"34 | "extern"35 | "fn"36 | "for"37 | "if"38 | "impl"39 | "in"40 | "let"41 | "loop"42 | "match"43 | "mod"44 | "move"45 | "pub"46 | "return"47 | "static"48 | "struct"49 | "super"50 | "trait"51 | "type"52 | "unsafe"53 | "use"54 | "where"55 | "while"56 | "abstract"57 | "become"58 | "box"59 | "do"60 | "final"61 | "gen"62 | "macro"63 | "override"64 | "priv"65 | "typeof"66 | "unsized"67 | "virtual"68 | "yield"69 | "try"70 ) || match text {71 "union" => next.is_some_and(|t| t.kind == TokenKind::Ident),72 "auto" => next.is_some_and(|t| t.text == "trait"),73 "raw" => next.is_some_and(|t| matches!(t.text, "const" | "mut")),74 "safe" => next.is_some_and(|t| t.text == "fn"),75 "default" => next.is_some_and(|t| {76 matches!(77 t.text,78 "fn" | "type" | "const" | "impl" | "unsafe" | "pub" | "async" | "extern"79 )80 }),81 _ => false,82 }83}84fn classify(tokens: &[Token<'_>]) -> Vec<Option<&'static str>> { warning: clippy::too_many_lines85 let mut following = vec![None; tokens.len()];86 let mut significant = None;87 for i in (0..tokens.len()).rev() {88 following[i] = significant;89 if !trivia(tokens[i].kind) {90 significant = Some(i);91 }92 }93 let mut classes = Vec::with_capacity(tokens.len());94 let mut previous_significant: Option<usize> = None;95 let mut attribute_depth = 0usize;96 let mut pending_attribute = false;97 for (i, token) in tokens.iter().enumerate() {98 let next = following[i].map(|j| &tokens[j]);99 let adjacent_next = tokens.get(i + 1);100 let adjacent_previous = i.checked_sub(1).map(|j| &tokens[j]);101 if token.kind == TokenKind::Pound && attribute_depth == 0 {102 pending_attribute = following[i].is_some_and(|j| {103 tokens[j].kind == TokenKind::OpenBracket104 || (tokens[j].kind == TokenKind::Bang105 && following[j].is_some_and(|k| tokens[k].kind == TokenKind::OpenBracket))106 });107 }108 if token.kind == TokenKind::OpenBracket && (pending_attribute || attribute_depth > 0) {109 attribute_depth += 1;110 pending_attribute = false;111 }112 let class = match token.kind {113 TokenKind::LineComment { doc_style } | TokenKind::BlockComment { doc_style, .. } => {114 Some(if doc_style.is_some() {115 "doccomment"116 } else {117 "comment"118 })119 }120 TokenKind::Frontmatter { .. } => Some("comment"),121 TokenKind::Literal {122 kind: LiteralKind::Int { .. } | LiteralKind::Float { .. },123 ..124 } => Some("number"),125 TokenKind::Literal { .. } => Some("string"),126 TokenKind::Lifetime { .. }127 | TokenKind::RawLifetime128 | TokenKind::UnknownPrefixLifetime => Some("lifetime"),129 TokenKind::Question => Some("question-mark"),130 TokenKind::Ident | TokenKind::RawIdent => {131 let macro_name = next.is_some_and(|t| t.kind == TokenKind::Bang)132 && !following[i]133 .and_then(|j| tokens.get(j + 1))134 .is_some_and(|t| t.kind == TokenKind::Eq);135 if previous_significant.is_some_and(|j| tokens[j].kind == TokenKind::Dollar) {136 Some("macro-nonterminal")137 } else if token.kind == TokenKind::RawIdent {138 macro_name.then_some("macro")139 } else {140 match token.text {141 "ref" | "mut" => Some("kw-2"),142 "const" if adjacent_previous.is_some_and(|t| t.kind == TokenKind::Star) => {143 Some("kw-2")144 }145 "self" | "Self" => Some("self"),146 "true" | "false" => Some("bool-val"),147 "Option" | "Result" => Some("prelude-ty"),148 "Some" | "None" | "Ok" | "Err" => Some("prelude-val"),149 text if keyword(text, next) => Some("kw"),150 _ => macro_name.then_some("macro"),151 }152 }153 }154 TokenKind::Bang155 if previous_significant.is_some_and(|j| classes[j] == Some("macro")) =>156 {157 Some("macro")158 }159 TokenKind::Dollar160 if next161 .is_some_and(|t| matches!(t.kind, TokenKind::Ident | TokenKind::RawIdent)) =>162 {163 Some("macro-nonterminal")164 }165 TokenKind::And166 if !adjacent_next.is_some_and(|t| {167 matches!(168 t.kind,169 TokenKind::And | TokenKind::Eq | TokenKind::Whitespace170 )171 }) && !adjacent_previous.is_some_and(|t| t.kind == TokenKind::And) =>172 {173 Some("kw-2")174 }175 TokenKind::Star176 if !adjacent_next177 .is_some_and(|t| matches!(t.kind, TokenKind::Eq | TokenKind::Whitespace)) =>178 {179 Some("kw-2")180 }181 _ => None,182 }183 .or_else(|| {184 ((attribute_depth > 0 || pending_attribute) && !trivia(token.kind)).then_some("attr")185 });186 classes.push(class);187 if token.kind == TokenKind::CloseBracket && attribute_depth > 0 {188 attribute_depth -= 1;189 }190 if !trivia(token.kind) {191 previous_significant = Some(i);192 }193 }194 classes195}196fn append(rows: &mut Vec<Vec<Fragment>>, text: &str, class: Option<&'static str>) {197 for (i, part) in text.split('\n').enumerate() {198 if i > 0 {199 rows.push(Vec::new());200 }201 if !part.is_empty() {202 rows.last_mut().expect("initial row").push(Fragment {203 text: part.into(),204 class,205 });206 }207 }208}209pub(super) fn lines(source: &str) -> Vec<Vec<Fragment>> {210 let source = source.replace("\r\n", "\n");211 let mut rows = vec![Vec::new()];212 let shebang = strip_shebang(&source).unwrap_or(0);213 append(&mut rows, &source[..shebang], Some("comment"));214 let mut offset = shebang;215 let tokens: Vec<_> = tokenize(&source[shebang..], FrontmatterAllowed::No)216 .map(|token| {217 let text = &source[offset..offset + token.len as usize];218 offset += token.len as usize;219 Token {220 kind: token.kind,221 text,222 }223 })224 .collect();225 for (token, class) in tokens.iter().zip(classify(&tokens)) {226 append(&mut rows, token.text, class);227 }228 rows229}230231#[cfg(test)]232mod tests {233 use super::*;234 #[test]235 fn underscore_patterns_and_inferred_types_use_rustdoc_keyword_class() {236 let source = "let _: Option<_> = None; let _name = 1; let s = \"_\"; // _\n";237 assert_eq!(238 classified(source, "kw")239 .iter()240 .filter(|text| text.as_str() == "_")241 .count(),242 2243 );244 assert!(!classified(source, "kw").contains(&"_name".into()));245 assert_eq!(classified(source, "string"), ["\"_\""]);246 assert_eq!(classified(source, "comment"), ["// _"]);247 }248 #[test]249 fn pointer_modifiers_and_logical_operators_keep_rustdoc_categories() {250 let source = "let p: *const u8; let r: &mut u8; if !true {} let x = a != b; let union = 1;";251 assert!(classified(source, "kw-2").contains(&"const".into()));252 assert!(classified(source, "kw-2").contains(&"mut".into()));253 assert!(!classified(source, "macro").contains(&"if".into()));254 assert!(!classified(source, "macro").contains(&"a".into()));255 assert!(!classified(source, "kw").contains(&"union".into()));256 }257 fn classified(source: &str, class: &str) -> Vec<String> {258 lines(source)259 .into_iter()260 .flatten()261 .filter(|fragment| fragment.class == Some(class))262 .map(|fragment| fragment.text)263 .collect()264 }265 #[test]266 fn keywords_literals_and_rustdoc_special_names_receive_native_classes() {267 let source = "pub fn demo<'a>(&mut self) -> Option<String> { let n = 42_u8; println!(\"<tag>&\"); Some(true) }";268 assert!(classified(source, "kw").contains(&"fn".into()));269 assert_eq!(classified(source, "string"), ["\"<tag>&\""]);270 assert_eq!(classified(source, "number"), ["42_u8"]);271 assert_eq!(classified(source, "prelude-ty"), ["Option"]);272 assert_eq!(classified(source, "prelude-val"), ["Some"]);273 assert_eq!(classified(source, "bool-val"), ["true"]);274 assert_eq!(classified(source, "lifetime"), ["'a"]);275 assert_eq!(classified(source, "self"), ["self"]);276 assert!(classified(source, "macro").contains(&"println".into()));277 assert!(classified(source, "kw-2").contains(&"mut".into()));278 }279 #[test]280 fn multiline_tokens_keep_their_class_and_original_line_boundaries() {281 let source = "let s = r##\"first\r\n/* not a comment */\nlast\"##;\n/* outer\n /* nested */ end */\nfn next() {}\n";282 let rows = lines(source);283 assert_eq!(rows.len(), 7);284 assert!(285 rows[1]286 .iter()287 .any(|fragment| fragment.class == Some("string")288 && fragment.text.contains("not a comment"))289 );290 assert!(291 rows[4]292 .iter()293 .any(|fragment| fragment.class == Some("comment")294 && fragment.text.contains("nested"))295 );296 assert!(297 rows[5]298 .iter()299 .any(|fragment| fragment.class == Some("kw") && fragment.text == "fn")300 );301 let roundtrip = rows302 .into_iter()303 .map(|row| {304 row.into_iter()305 .map(|fragment| fragment.text)306 .collect::<String>()307 })308 .collect::<Vec<_>>()309 .join("\n");310 assert_eq!(roundtrip, source.replace("\r\n", "\n"));311 }312 #[test]313 fn attributes_raw_identifiers_chars_and_macro_metavariables_are_distinct() {314 let source = "#![allow(dead_code)]\n#[cfg(any(feature = \"x\", feature = \"y\"))]\nfn r#match<'a>() { let c = '한'; let p = c\"cstr\"; let b = br#\"bytes\"#; }\nmacro_rules! m { ($x:expr) => { $x }; }\n";315 assert!(classified(source, "attr").contains(&"allow".into()));316 assert!(!classified(source, "kw").contains(&"r#match".into()));317 assert!(classified(source, "string").contains(&"'한'".into()));318 assert!(classified(source, "string").contains(&"c\"cstr\"".into()));319 assert!(classified(source, "string").contains(&"br#\"bytes\"#".into()));320 assert!(classified(source, "macro-nonterminal").contains(&"x".into()));321 }322 #[test]323 fn arbitrary_unicode_html_and_incomplete_input_preserve_text() {324 for source in [325 "",326 "\n",327 "\u{feff}#!/bin/rust\nfn 한글() {}\n",328 "/// docs <script>&\nlet s = \"unfinished\n한글",329 "/* unfinished\n nested /* */",330 "r###\"unterminated\n#",331 "'a '\\n' & && *= != # \u{1f600}",332 ] {333 let text = lines(source)334 .into_iter()335 .map(|row| {336 row.into_iter()337 .map(|fragment| fragment.text)338 .collect::<String>()339 })340 .collect::<Vec<_>>()341 .join("\n");342 assert_eq!(text, source.replace("\r\n", "\n"));343 }344 }345}346Findings in this file§
warning clippy::too_many_lines §
warning clippy::too_many_lines §
Targets (1)
cargo-clippy-html (bin)
Primary: src/render/highlight.rs:84:1
help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.100.0/index.html#too_many_lines
note: `-W clippy::too-many-lines` implied by `-W clippy::pedantic`
help: to override `-W clippy::pedantic` add `#[allow(clippy::too_many_lines)]`
Rendered diagnostic (text)
warning: this function has too many lines (110/100)
--> src/render/highlight.rs:84:1
|
84 | fn classify(tokens: &[Token<'_>]) -> Vec<Option<&'static str>> {
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
= help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.100.0/index.html#too_many_lines
= note: `-W clippy::too-many-lines` implied by `-W clippy::pedantic`
= help: to override `-W clippy::pedantic` add `#[allow(clippy::too_many_lines)]`