Steps for html-to-md script

wget -r -m -p -x --user-agent="Mozilla/5.0" https://www.websiteurlhere.com

git init
git add .
git commit -m 'init commit'

vimdiff  new-pages.txt live-pages.txt -c TOhtml -c 'w! diff.html' -c 'qa!'

##############################
#         READ ME
###

# This script loops through every page in the directory (except /index and /template.index),
# pulling the title tag and description from the head,
# with the page content in the body then putting them around
# in the template file.

# **CAUTION**
# This does require a few edits, which will be listed below.

# Read the outputs to follow along with the script.

# Let's see how it's done!

# Requiring dependencies
puts "Requiring dependencies"
require 'nokogiri'
require 'reverse_markdown'
require 'find'
require 'date'
require 'fileutils'
require 'json' 

# Define CSS selectors and other configuration settings at the top for easy modification
CONFIG = {
  ignore_files: ['./index.html', './indexNEW.html'],
  ignore_folders: ['/assets/', '.git', '.htaccess'],
  blog_selector: ".main-content-container.blog",
  title: 'title',
  description: 'meta[name="description"]',
  category: 'a[rel="category tag"]',
  h1: 'h1',
  json_ld: "script[type='application/ld+json']",
  content: 'main.main-content-container',
  remove_selectors: [
    '.extra-selector-1',  # Add your extra selectors here
    '.extra-selector-2'   # Add more selectors as needed
  ],
  # Both of these need to be changed if scraping categories from the blog snippets
  # Path to main blog page
  snippet_file_path: './blog/index.html',  
  # Base domain 
  domain_url: 'https://www.EXAMPLE.com'
}

# Read and Parse the Snippet File to Build the Blog Categories Map
snippet_file = File.read(CONFIG[:snippet_file_path])
puts "Snippet file read successfully"
snippet_doc = Nokogiri::HTML(snippet_file)
puts "Snippet file parsed successfully"

blog_categories = {}
snippet_doc.css('article.post').each do |post|
  # Normalize URL by removing domain, trailing slashes, and whitespace
  url = post.at_css('a')["href"].gsub(CONFIG[:domain_url], '').sub(%r{/*$}, '').strip + '/'
  category = post["category"]
  blog_categories[url] = category
end
puts "Blog categories mapped successfully"

# Method to process each file
def process_file(f, config, blog_categories)
  puts "---"
  puts "Processing file: #{f}"

  file = File.read(f)
  puts "File read successfully"

  doc = Nokogiri::HTML(file)
  puts "File parsed successfully"
  puts "Beginning DOM interaction"

  # Extract title, ensuring it handles nil gracefully
  title = doc.at_css(config[:title])&.text&.strip || ""
  puts "Found title tag: #{title}"

  # Extract description
  description_tag = doc.at_css(config[:description])
  description = description_tag ? description_tag["content"].strip : ""
  puts "Found header metadata: title=#{title}, description=#{description}"

  # Determine if this is a blog page
  is_blog = doc.at_css(config[:blog_selector])
  if is_blog
    puts 'Processing as BLOG PAGE'

    # Generate permalink with consistent trailing slash and strip whitespace
    permalink = f.gsub("./", "/").gsub("/index.html", "").sub(%r{/*$}, '').strip + '/'
    puts "Generated permalink for post: #{permalink}"

    # Primary category check using config selector
    category = doc.at_css(config[:category])&.text&.strip
    if category.nil? || category.empty?
      # Fallback to blog snippet if primary selector didn’t find anything
      puts "No category set, looking for category in blog snippet..."
      category = blog_categories[permalink] || "Uncategorized"
      puts "Category found from blog snippet: #{category}"
    else
      puts "Category found on page: #{category}"
    end

    # Extract h1
    h1 = doc.at_css(config[:h1])&.text&.strip || ""
    doc.at_css(config[:h1])&.remove
    puts "Found H1: #{h1}"

    # Directly attempt to extract datePublished
    date = Date.today  # Default if datePublished isn’t found
    json_ld_scripts = doc.css("script[type='application/ld+json']")

    date_found = false  # Track whether a date was found

    json_ld_scripts.each do |script|
      begin
        json_content = JSON.parse(script.text) rescue nil
        if json_content && json_content["@type"] == "BlogPosting" && json_content["datePublished"]
          date = json_content["datePublished"]
          date_found = true
          puts "Found datePublished in schema: #{date}"  # Message confirming date was found
          break
        end
      rescue JSON::ParserError => e
        puts "JSON parsing error: #{e.message}"
      end
    end

    # If no date was found in the schema, confirm it’s using today’s date
    puts "No datePublished found in schema. Using today's date: #{date}" unless date_found

    # Remove additional unwanted selectors from the content
    config[:remove_selectors].each do |selector|
      doc.css(selector).each(&:remove)
    end
    puts "Removed additional unwanted content"

    content = doc.at_css(config[:content]).inner_html
    return if content.nil?

    markdown = ReverseMarkdown.convert(content.to_s)
    puts "Converted content to Markdown"

    # Extract and create the front matter
    frontMatter = <<~FRONTMATTER
      ---
      title: >
        #{h1}
      layout: post
      date: >
        #{date}
      titletag: >
        #{title}
      description: >
        #{description}
      permalink: >
        #{permalink}
      sitemap: true
      categories:
        - #{category}
      ---
    FRONTMATTER
    puts "Saved front matter"

    newPage = frontMatter + markdown
    puts "Combined front matter and content"

    # Generate file paths and write the Markdown file
    filename = f.split('/')[-2]
    mdF = "./_blogposts/#{filename}.md"
    puts "Markdown file path: #{mdF}"

    directory = mdF.gsub(mdF.split('/').last, "")
    puts "Making directory: #{directory}"

    FileUtils.mkdir_p(directory)
    puts "Directory created"

    File.write(mdF, newPage)
    puts "Written to file: #{mdF}"

    FileUtils.rm_rf(f)
    puts "Removed old file: #{f}"
  else
    puts 'Processing as regular PAGE'

    # Remove additional unwanted selectors from the content
    config[:remove_selectors].each do |selector|
      doc.css(selector).each(&:remove)
    end
    puts "Removed additional unwanted content"

    # Convert content to Markdown and create the Markdown file
    content = doc.at_css(config[:content]).inner_html
    if content.nil?
      puts "Content not found using selector: #{config[:content]}"
      return
    else
      puts "Found content"
    end

    markdown = ReverseMarkdown.convert(content.to_s)
    puts "Converted content to Markdown"

    # Extract and create the front matter
    frontMatter = <<~FRONTMATTER
      ---
      layout: page
      title: >
        #{f.gsub('./', '').gsub('.html', '').gsub('-', ' ').split('/').map(&:capitalize).join(' ')}
      titletag: >
        #{title}
      description: >
        #{description}
      permalink: >
        #{f.gsub("./","/").gsub("/index.html","/")}
      titlebar: >
      sitemap: true
      ---
    FRONTMATTER
    puts "Saved front matter"

    newPage = frontMatter + markdown
    puts "Combined front matter and content"

    # Generate filename for pages by replacing slashes with dashes and removing index.html
    filename = f.gsub('./', '').gsub('/index.html', '').gsub('.html', '').gsub('/', '-')
    mdF = "./_pages/#{filename}.md"
    puts "Markdown file path: #{mdF}"

    directory = mdF.gsub(mdF.split('/').last, "")
    puts "Making directory: #{directory}"

    FileUtils.mkdir_p(directory)
    puts "Directory created"

    File.write(mdF, newPage)
    puts "Written to file: #{mdF}"

    FileUtils.rm_rf(f)
    puts "Removed old file: #{f}"
  end
rescue => e
  puts "Error processing file #{f}: #{e.message}"
  puts e.backtrace.join("\n")
end

# Loop through each HTML file and process it
puts "Beginning loop of every file"
Find.find("./") do |f|
  if CONFIG[:ignore_files].include?(f)
    puts "Ignored file: #{f}"
  elsif CONFIG[:ignore_folders].any? { |folder| f.include?(folder) }
    puts "Ignored folder content: #{f}"
  elsif f.include?('.html')
    process_file(f, CONFIG, blog_categories)
  else
    puts "Not an HTML file: #{f}"
  end
end

exec "find ./ -empty -type d -delete"
puts "Removed all empty directories"