website/scripts: Add docsmg migration tool (#10658)

* add docsmg tool

* moved to the correct scripts directory

* removed test files

website/scripts/docsmg/src/generate.rs

@@ -0,0 +1,31 @@
+use std::path::PathBuf;
+
+use crate::{migratefile::read_migrate_file, recurse_directory};
+
+pub fn generate(migratefile: Option<PathBuf>, migrate_path: PathBuf) {
+    // if there is a migrate file, read it and get the paths from the left side
+    let paths: Vec<PathBuf> = match migratefile {
+        Some(i) => {
+            let contents = read_migrate_file(i);
+            if let Ok(contents) = contents {
+                contents.iter().map(|x| x.0.clone()).collect()
+            } else {
+                vec![]
+            }
+        }
+        None => {
+            vec![]
+        }
+    };
+    // get rid of paths already in the specified migrate file
+    let paths: Vec<PathBuf> = recurse_directory(migrate_path.clone())
+        .iter()
+        .filter(|x| !paths.contains(x))
+        .filter_map(|x| x.strip_prefix(migrate_path.clone()).ok())
+        .map(|x| x.to_path_buf())
+        .collect();
+
+    for path in paths {
+        println!("{} -> ", path.display());
+    }
+}

website/scripts/docsmg/src/main.rs

@@ -0,0 +1,83 @@
+use std::{fs, path::PathBuf};
+
+use clap::{Parser, Subcommand};
+
+mod generate;
+mod migrate;
+mod migratefile;
+mod r#move;
+
+#[derive(Parser)]
+struct Cli {
+    #[arg(long, env, default_value = "./")]
+    migrate_path: PathBuf,
+
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+    Move {
+        old_path: PathBuf,
+        new_path: PathBuf,
+    },
+    Migrate {
+        #[arg(long, name = "FILE", default_value = "./migratefile")]
+        migratefile: PathBuf,
+
+        #[arg(short, long)]
+        quiet: bool,
+    },
+    Unmigrate {
+        #[arg(long, name = "FILE", default_value = "./migratefile")]
+        migratefile: PathBuf,
+
+        #[arg(short, long)]
+        quiet: bool,
+    },
+    Generate {
+        #[arg(long, name = "FILE")]
+        migratefile: Option<PathBuf>,
+    },
+}
+
+fn main() {
+    let cli = Cli::parse();
+
+    match cli.command {
+        Commands::Move { old_path, new_path } => r#move::r#move(old_path, new_path),
+        Commands::Migrate { migratefile, quiet } => {
+            migrate::migrate(quiet, migratefile, cli.migrate_path)
+        }
+        Commands::Unmigrate { migratefile, quiet } => {
+            migrate::unmigrate(quiet, migratefile, cli.migrate_path)
+        }
+        Commands::Generate { migratefile } => generate::generate(migratefile, cli.migrate_path),
+    }
+}
+
+fn recurse_directory(path: PathBuf) -> Vec<PathBuf> {
+    let paths = fs::read_dir(path).expect("path to exist");
+    let mut final_paths = vec![];
+    for path in paths {
+        match path {
+            Ok(path) => {
+                if !path.path().is_file() && !path.path().is_dir() {
+                    continue;
+                } // dont go any further if not a file or directory
+                let is_dir = path.path().is_dir();
+                let path = path.path();
+
+                if is_dir {
+                    let mut paths = recurse_directory(path);
+                    final_paths.append(&mut paths);
+                } else {
+                    final_paths.push(path);
+                }
+            }
+            _ => {}
+        }
+    }
+    final_paths
+}

website/scripts/docsmg/src/migrate.rs

@@ -0,0 +1,212 @@
+use std::{
+    collections::HashMap, ffi::OsStr, fmt::format, fs::{read_to_string, write}, path::{Component, Path, PathBuf}
+};
+
+use colored::Colorize;
+use lazy_static::lazy_static;
+
+use crate::{migratefile::read_migrate_file, recurse_directory};
+
+pub fn migrate(quiet: bool, migratefile: PathBuf, migrate_path: PathBuf) {
+    if !quiet {
+        println!("Reading migrate file");
+    }
+    let files = read_migrate_file(migratefile);
+
+    let files = match files {
+        Ok(i) => {
+            if !quiet {
+                println!("{}", "Success".green());
+            }
+            i
+        }
+        Err(_) => {
+            println!("{}: Could not read migrate file", "Error".red());
+            return;
+        }
+    };
+
+    replace_links(migrate_path.clone(), files.clone());
+    let successful_moves = move_files(quiet, migrate_path.clone(), files);
+    add_redirects(successful_moves.clone(), migrate_path.clone());
+}
+
+pub fn unmigrate(quiet: bool, migratefile: PathBuf, migrate_path: PathBuf) {
+    if !quiet {
+        println!("Reading migrate file");
+    }
+    let files = read_migrate_file(migratefile);
+
+    let files = match files {
+        Ok(i) => {
+            if !quiet {
+                println!("{}", "Success".green());
+            }
+            i
+        }
+        Err(_) => {
+            println!("{}: Could not read migrate file", "Error".red());
+            return;
+        }
+    };
+
+    let files: Vec<(PathBuf, PathBuf)> = files.iter().map(|x| (x.1.clone(), x.0.clone())).collect(); //switch files to reverse a migration
+    replace_links(migrate_path.clone(), files.clone());
+    let successful_moves = move_files(quiet, migrate_path.clone(), files);
+    let successful_moves: Vec<(PathBuf, PathBuf)> = successful_moves
+        .iter()
+        .map(|x| (x.1.clone(), x.0.clone()))
+        .collect(); //switch files to reverse a migration
+    remove_redirects(successful_moves, migrate_path);
+}
+
+fn move_files(
+    quiet: bool,
+    migrate_path: PathBuf,
+    files: Vec<(PathBuf, PathBuf)>,
+) -> Vec<(PathBuf, PathBuf)> {
+    let mut successful_moves = vec![];
+    for file in files {
+        if !quiet {
+            print!("{} -> {} : ", file.0.display(), file.1.display());
+        }
+        let rename: anyhow::Result<()> = (|| {
+            let old_file = migrate_path.join(&file.0);
+            let new_file = migrate_path.join(&file.1);
+            std::fs::create_dir_all(&new_file.parent().expect("files to have a parent"))?;
+            std::fs::rename(&old_file, &new_file)?;
+            Ok(())
+        })();
+        match rename {
+            Ok(_) => {
+                if !quiet {
+                    println!("{}", "Success".green());
+                }
+                successful_moves.push(file);
+            }
+            Err(_) => println!(
+                "{}: Could not move file {}",
+                "Error".red(),
+                file.0.display()
+            ),
+        };
+    }
+    successful_moves
+}
+
+fn replace_links(migrate_path: PathBuf, successful_moves: Vec<(PathBuf, PathBuf)>) {
+    lazy_static! {
+        static ref find_link: regex::Regex =
+            regex::Regex::new(r"\[(?<a>.*)\]\((?<b>.*)\)").unwrap();
+    }
+    let files = recurse_directory(migrate_path.clone());
+
+    for file in files {
+        let relative_file = file.strip_prefix(migrate_path.clone()).unwrap().to_path_buf();
+        let mut contents = match read_to_string(file.clone()) {
+            Ok(i) => i,
+            Err(_) => continue,
+        };
+        let mut replace = vec![];
+        for successful_move in &successful_moves {
+            if migrate_path.join(successful_move.0.clone()).canonicalize().unwrap() 
+                == file.clone().canonicalize().unwrap() {
+                continue;
+            }
+            let new_successful_move_from = make_path_relative(successful_move.0.clone(), relative_file.clone());
+            let new_successful_move_to = make_path_relative(successful_move.1.clone(), relative_file.clone());
+            replace.push((new_successful_move_from, new_successful_move_to));
+        }
+        for i in replace {
+            println!("{} : {} -> {}", file.display(), i.0.display(), i.1.display());
+            contents = contents.replace(&format!("({})", i.0.display()), &format!("({})", i.1.display()));
+        }
+        write(file, contents).unwrap();
+    }
+}
+
+fn make_path_relative(path: PathBuf, relative_to: PathBuf) -> PathBuf {
+    let mut subdirs = 0;
+    let path_components = path.components().collect::<Vec<_>>();
+    let relative_to_components = relative_to.components().collect::<Vec<_>>();
+    loop {
+        if path_components.len() <= subdirs {
+            break;
+        } else if path_components[subdirs]
+            != relative_to_components[subdirs]
+        {
+            break;
+        }
+        subdirs += 1;
+    }
+    let new_path = &path_components[subdirs..]
+        .iter()
+        .collect::<PathBuf>();
+    let backouts =
+        (0..relative_to_components.len() - subdirs - 1)
+            .map(|_| PathBuf::from(".."))
+            .reduce(|acc, e| acc.join(e))
+            .unwrap_or(PathBuf::from(""));
+    //println!("{}, {}", relative_to_components.len() - subdirs - 1, backouts.display());
+    let new_path = backouts.join(new_path);
+    let new_path = if new_path
+        .to_string_lossy()
+        .to_string()
+        .chars()
+        .next()
+        .unwrap()
+        != '.'
+    {
+        PathBuf::from(".").join(new_path)
+    } else {
+        new_path
+    };
+
+    let new_path = if new_path.file_name() == Some(OsStr::new("index.md")) || new_path.file_name() == Some(OsStr::new("index.mdx")) {
+        new_path.parent().unwrap().to_path_buf()
+    } else {
+        new_path
+    };
+        
+
+    new_path
+}
+
+fn add_redirects(successful_moves: Vec<(PathBuf, PathBuf)>, migrate_path: PathBuf) {
+    let redirects = generate_redirects(successful_moves);
+    let netlify_path = migrate_path.parent().unwrap().join("netlify.toml");
+    let mut netlify_contents = read_to_string(netlify_path.clone()).unwrap();
+    for redirect in redirects {
+        netlify_contents.push_str(&redirect);
+    }
+    std::fs::write(netlify_path, netlify_contents).unwrap();
+}
+
+fn remove_redirects(successful_moves: Vec<(PathBuf, PathBuf)>, migrate_path: PathBuf) {
+    let redirects = generate_redirects(successful_moves);
+    let netlify_path = migrate_path.parent().unwrap().join("netlify.toml");
+    let mut netlify_contents = read_to_string(netlify_path.clone()).unwrap();
+    for redirect in redirects {
+        netlify_contents = netlify_contents.replace(&redirect, "");
+    }
+    std::fs::write(netlify_path, netlify_contents).unwrap();
+}
+
+fn generate_redirects(successful_moves: Vec<(PathBuf, PathBuf)>) -> Vec<String> {
+    successful_moves
+        .iter()
+        .map(|x| {
+            format!(
+                "
+[[redirects]]
+  from = \"{}\"
+  to = \"{}\"
+  status = 301
+  force = true
+",
+                x.0.display(),
+                x.1.display()
+            )
+        })
+        .collect()
+}

website/scripts/docsmg/src/migratefile.rs

@@ -0,0 +1,21 @@
+use std::{fs::read_to_string, path::PathBuf};
+
+pub fn read_migrate_file(file: PathBuf) -> anyhow::Result<Vec<(PathBuf, PathBuf)>> {
+    let contents = read_to_string(file)?;
+    let lines: Vec<String> = contents
+        .split('\n')
+        .map(|x| x.to_owned())
+        .filter(|x| x != "")
+        .collect();
+    let migrations = lines
+        .iter()
+        .filter_map(|x| x.split_once(" -> "))
+        .map(|x| {
+            (
+                x.0.parse().expect("a valid path"),
+                x.1.parse().expect("a valid path"),
+            )
+        })
+        .collect::<Vec<_>>();
+    Ok(migrations)
+}

website/scripts/docsmg/src/move.rs

@@ -0,0 +1,23 @@
+use std::path::PathBuf;
+
+use crate::recurse_directory;
+
+pub fn r#move(old_path: PathBuf, new_path: PathBuf) {
+    let is_dir = old_path.is_dir();
+    if is_dir {
+        let paths = recurse_directory(old_path.clone());
+        for path in paths {
+            let raw_path = path
+                .strip_prefix(old_path.clone())
+                .expect("path to be within old path");
+            let new_path = new_path.join(raw_path);
+            println!("{} -> {}", path.display(), new_path.display());
+        }
+    } else {
+        println!(
+            "{} -> {}",
+            old_path.to_string_lossy(),
+            new_path.to_string_lossy()
+        );
+    }
+}