Skip to main content

Source src/render/highlight.rs

1//! Lexical source fragments using rustdoc's CSS classes, never raw HTML.2// Classification follows rustdoc html/highlight.rs; see HIGHLIGHTING.md for3// source references, selected MIT licensing and compiler-context limitations.4use ra_ap_rustc_lexer::{FrontmatterAllowed, LiteralKind, TokenKind, strip_shebang, tokenize};56pub(super) struct Fragment {7    pub text: String,8    pub class: Option<&'static str>,9}1011struct Token<'a> {12    kind: TokenKind,13    text: &'a str,14}15fn trivia(kind: TokenKind) -> bool {16    matches!(17        kind,18        TokenKind::Whitespace | TokenKind::LineComment { .. } | TokenKind::BlockComment { .. }19    )20}21fn keyword(text: &str, next: Option<&Token<'_>>) -> bool {22    matches!(23        text,24        "_" | "as"25            | "async"26            | "await"27            | "break"28            | "const"29            | "continue"30            | "crate"31            | "dyn"32            | "else"33            | "enum"34            | "extern"35            | "fn"36            | "for"37            | "if"38            | "impl"39            | "in"40            | "let"41            | "loop"42            | "match"43            | "mod"44            | "move"45            | "pub"46            | "return"47            | "static"48            | "struct"49            | "super"50            | "trait"51            | "type"52            | "unsafe"53            | "use"54            | "where"55            | "while"56            | "abstract"57            | "become"58            | "box"59            | "do"60            | "final"61            | "gen"62            | "macro"63            | "override"64            | "priv"65            | "typeof"66            | "unsized"67            | "virtual"68            | "yield"69            | "try"70    ) || match text {71        "union" => next.is_some_and(|t| t.kind == TokenKind::Ident),72        "auto" => next.is_some_and(|t| t.text == "trait"),73        "raw" => next.is_some_and(|t| matches!(t.text, "const" | "mut")),74        "safe" => next.is_some_and(|t| t.text == "fn"),75        "default" => next.is_some_and(|t| {76            matches!(77                t.text,78                "fn" | "type" | "const" | "impl" | "unsafe" | "pub" | "async" | "extern"79            )80        }),81        _ => false,82    }83}84fn classify(tokens: &[Token<'_>]) -> Vec<Option<&'static str>> { warning: clippy::too_many_lines85    let mut following = vec![None; tokens.len()];86    let mut significant = None;87    for i in (0..tokens.len()).rev() {88        following[i] = significant;89        if !trivia(tokens[i].kind) {90            significant = Some(i);91        }92    }93    let mut classes = Vec::with_capacity(tokens.len());94    let mut previous_significant: Option<usize> = None;95    let mut attribute_depth = 0usize;96    let mut pending_attribute = false;97    for (i, token) in tokens.iter().enumerate() {98        let next = following[i].map(|j| &tokens[j]);99        let adjacent_next = tokens.get(i + 1);100        let adjacent_previous = i.checked_sub(1).map(|j| &tokens[j]);101        if token.kind == TokenKind::Pound && attribute_depth == 0 {102            pending_attribute = following[i].is_some_and(|j| {103                tokens[j].kind == TokenKind::OpenBracket104                    || (tokens[j].kind == TokenKind::Bang105                        && following[j].is_some_and(|k| tokens[k].kind == TokenKind::OpenBracket))106            });107        }108        if token.kind == TokenKind::OpenBracket && (pending_attribute || attribute_depth > 0) {109            attribute_depth += 1;110            pending_attribute = false;111        }112        let class = match token.kind {113            TokenKind::LineComment { doc_style } | TokenKind::BlockComment { doc_style, .. } => {114                Some(if doc_style.is_some() {115                    "doccomment"116                } else {117                    "comment"118                })119            }120            TokenKind::Frontmatter { .. } => Some("comment"),121            TokenKind::Literal {122                kind: LiteralKind::Int { .. } | LiteralKind::Float { .. },123                ..124            } => Some("number"),125            TokenKind::Literal { .. } => Some("string"),126            TokenKind::Lifetime { .. }127            | TokenKind::RawLifetime128            | TokenKind::UnknownPrefixLifetime => Some("lifetime"),129            TokenKind::Question => Some("question-mark"),130            TokenKind::Ident | TokenKind::RawIdent => {131                let macro_name = next.is_some_and(|t| t.kind == TokenKind::Bang)132                    && !following[i]133                        .and_then(|j| tokens.get(j + 1))134                        .is_some_and(|t| t.kind == TokenKind::Eq);135                if previous_significant.is_some_and(|j| tokens[j].kind == TokenKind::Dollar) {136                    Some("macro-nonterminal")137                } else if token.kind == TokenKind::RawIdent {138                    macro_name.then_some("macro")139                } else {140                    match token.text {141                        "ref" | "mut" => Some("kw-2"),142                        "const" if adjacent_previous.is_some_and(|t| t.kind == TokenKind::Star) => {143                            Some("kw-2")144                        }145                        "self" | "Self" => Some("self"),146                        "true" | "false" => Some("bool-val"),147                        "Option" | "Result" => Some("prelude-ty"),148                        "Some" | "None" | "Ok" | "Err" => Some("prelude-val"),149                        text if keyword(text, next) => Some("kw"),150                        _ => macro_name.then_some("macro"),151                    }152                }153            }154            TokenKind::Bang155                if previous_significant.is_some_and(|j| classes[j] == Some("macro")) =>156            {157                Some("macro")158            }159            TokenKind::Dollar160                if next161                    .is_some_and(|t| matches!(t.kind, TokenKind::Ident | TokenKind::RawIdent)) =>162            {163                Some("macro-nonterminal")164            }165            TokenKind::And166                if !adjacent_next.is_some_and(|t| {167                    matches!(168                        t.kind,169                        TokenKind::And | TokenKind::Eq | TokenKind::Whitespace170                    )171                }) && !adjacent_previous.is_some_and(|t| t.kind == TokenKind::And) =>172            {173                Some("kw-2")174            }175            TokenKind::Star176                if !adjacent_next177                    .is_some_and(|t| matches!(t.kind, TokenKind::Eq | TokenKind::Whitespace)) =>178            {179                Some("kw-2")180            }181            _ => None,182        }183        .or_else(|| {184            ((attribute_depth > 0 || pending_attribute) && !trivia(token.kind)).then_some("attr")185        });186        classes.push(class);187        if token.kind == TokenKind::CloseBracket && attribute_depth > 0 {188            attribute_depth -= 1;189        }190        if !trivia(token.kind) {191            previous_significant = Some(i);192        }193    }194    classes195}196fn append(rows: &mut Vec<Vec<Fragment>>, text: &str, class: Option<&'static str>) {197    for (i, part) in text.split('\n').enumerate() {198        if i > 0 {199            rows.push(Vec::new());200        }201        if !part.is_empty() {202            rows.last_mut().expect("initial row").push(Fragment {203                text: part.into(),204                class,205            });206        }207    }208}209pub(super) fn lines(source: &str) -> Vec<Vec<Fragment>> {210    let source = source.replace("\r\n", "\n");211    let mut rows = vec![Vec::new()];212    let shebang = strip_shebang(&source).unwrap_or(0);213    append(&mut rows, &source[..shebang], Some("comment"));214    let mut offset = shebang;215    let tokens: Vec<_> = tokenize(&source[shebang..], FrontmatterAllowed::No)216        .map(|token| {217            let text = &source[offset..offset + token.len as usize];218            offset += token.len as usize;219            Token {220                kind: token.kind,221                text,222            }223        })224        .collect();225    for (token, class) in tokens.iter().zip(classify(&tokens)) {226        append(&mut rows, token.text, class);227    }228    rows229}230231#[cfg(test)]232mod tests {233    use super::*;234    #[test]235    fn underscore_patterns_and_inferred_types_use_rustdoc_keyword_class() {236        let source = "let _: Option<_> = None; let _name = 1; let s = \"_\"; // _\n";237        assert_eq!(238            classified(source, "kw")239                .iter()240                .filter(|text| text.as_str() == "_")241                .count(),242            2243        );244        assert!(!classified(source, "kw").contains(&"_name".into()));245        assert_eq!(classified(source, "string"), ["\"_\""]);246        assert_eq!(classified(source, "comment"), ["// _"]);247    }248    #[test]249    fn pointer_modifiers_and_logical_operators_keep_rustdoc_categories() {250        let source = "let p: *const u8; let r: &mut u8; if !true {} let x = a != b; let union = 1;";251        assert!(classified(source, "kw-2").contains(&"const".into()));252        assert!(classified(source, "kw-2").contains(&"mut".into()));253        assert!(!classified(source, "macro").contains(&"if".into()));254        assert!(!classified(source, "macro").contains(&"a".into()));255        assert!(!classified(source, "kw").contains(&"union".into()));256    }257    fn classified(source: &str, class: &str) -> Vec<String> {258        lines(source)259            .into_iter()260            .flatten()261            .filter(|fragment| fragment.class == Some(class))262            .map(|fragment| fragment.text)263            .collect()264    }265    #[test]266    fn keywords_literals_and_rustdoc_special_names_receive_native_classes() {267        let source = "pub fn demo<'a>(&mut self) -> Option<String> { let n = 42_u8; println!(\"<tag>&\"); Some(true) }";268        assert!(classified(source, "kw").contains(&"fn".into()));269        assert_eq!(classified(source, "string"), ["\"<tag>&\""]);270        assert_eq!(classified(source, "number"), ["42_u8"]);271        assert_eq!(classified(source, "prelude-ty"), ["Option"]);272        assert_eq!(classified(source, "prelude-val"), ["Some"]);273        assert_eq!(classified(source, "bool-val"), ["true"]);274        assert_eq!(classified(source, "lifetime"), ["'a"]);275        assert_eq!(classified(source, "self"), ["self"]);276        assert!(classified(source, "macro").contains(&"println".into()));277        assert!(classified(source, "kw-2").contains(&"mut".into()));278    }279    #[test]280    fn multiline_tokens_keep_their_class_and_original_line_boundaries() {281        let source = "let s = r##\"first\r\n/* not a comment */\nlast\"##;\n/* outer\n /* nested */ end */\nfn next() {}\n";282        let rows = lines(source);283        assert_eq!(rows.len(), 7);284        assert!(285            rows[1]286                .iter()287                .any(|fragment| fragment.class == Some("string")288                    && fragment.text.contains("not a comment"))289        );290        assert!(291            rows[4]292                .iter()293                .any(|fragment| fragment.class == Some("comment")294                    && fragment.text.contains("nested"))295        );296        assert!(297            rows[5]298                .iter()299                .any(|fragment| fragment.class == Some("kw") && fragment.text == "fn")300        );301        let roundtrip = rows302            .into_iter()303            .map(|row| {304                row.into_iter()305                    .map(|fragment| fragment.text)306                    .collect::<String>()307            })308            .collect::<Vec<_>>()309            .join("\n");310        assert_eq!(roundtrip, source.replace("\r\n", "\n"));311    }312    #[test]313    fn attributes_raw_identifiers_chars_and_macro_metavariables_are_distinct() {314        let source = "#![allow(dead_code)]\n#[cfg(any(feature = \"x\", feature = \"y\"))]\nfn r#match<'a>() { let c = '한'; let p = c\"cstr\"; let b = br#\"bytes\"#; }\nmacro_rules! m { ($x:expr) => { $x }; }\n";315        assert!(classified(source, "attr").contains(&"allow".into()));316        assert!(!classified(source, "kw").contains(&"r#match".into()));317        assert!(classified(source, "string").contains(&"'한'".into()));318        assert!(classified(source, "string").contains(&"c\"cstr\"".into()));319        assert!(classified(source, "string").contains(&"br#\"bytes\"#".into()));320        assert!(classified(source, "macro-nonterminal").contains(&"x".into()));321    }322    #[test]323    fn arbitrary_unicode_html_and_incomplete_input_preserve_text() {324        for source in [325            "",326            "\n",327            "\u{feff}#!/bin/rust\nfn 한글() {}\n",328            "/// docs <script>&\nlet s = \"unfinished\n한글",329            "/* unfinished\n nested /* */",330            "r###\"unterminated\n#",331            "'a '\\n' & && *= != # \u{1f600}",332        ] {333            let text = lines(source)334                .into_iter()335                .map(|row| {336                    row.into_iter()337                        .map(|fragment| fragment.text)338                        .collect::<String>()339                })340                .collect::<Vec<_>>()341                .join("\n");342            assert_eq!(text, source.replace("\r\n", "\n"));343        }344    }345}346

Findings in this file§

warning clippy::too_many_lines §

this function has too many lines (110/100)

src/render/highlight.rs:84:1

Clippy · Category: pedantic · cargo-clippy-html

Official Clippy documentation

Targets (1)
  • cargo-clippy-html (bin)

Primary: src/render/highlight.rs:84:1

fn classify(tokens: &[Token<'_>]) -> Vec<Option<&'static str>> {

help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.100.0/index.html#too_many_lines

note: `-W clippy::too-many-lines` implied by `-W clippy::pedantic`

help: to override `-W clippy::pedantic` add `#[allow(clippy::too_many_lines)]`

Rendered diagnostic (text)
warning: this function has too many lines (110/100)
  --> src/render/highlight.rs:84:1
   |
84 | fn classify(tokens: &[Token<'_>]) -> Vec<Option<&'static str>> {
   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |
   = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.100.0/index.html#too_many_lines
   = note: `-W clippy::too-many-lines` implied by `-W clippy::pedantic`
   = help: to override `-W clippy::pedantic` add `#[allow(clippy::too_many_lines)]`