Forem: Tech Believers

Building a Heading Tags Analyzer: HTML Parsing and Hierarchy Validation

Tech Believers — Sun, 07 Dec 2025 15:54:46 +0000

Introduction

Heading tags (H1-H6) provide semantic structure to web content, but many sites have broken hierarchies that confuse search engines and hurt SEO. In this guide, we'll build a comprehensive heading tags analyzer that extracts, validates, and visualizes heading structure.

Understanding Heading Tag Hierarchy

Proper heading structure follows a logical outline:

<h1>Main Title</h1>
  <h2>Section 1</h2>
    <h3>Subsection 1.1</h3>
    <h3>Subsection 1.2</h3>
  <h2>Section 2</h2>
    <h3>Subsection 2.1</h3>
      <h4>Detail 2.1.1</h4>

This creates a tree structure that both search engines and screen readers can navigate.

Building the Analyzer

Step 1: Extracting Headings

function extractHeadings(html) {
    const parser = new DOMParser();
    const doc = parser.parseFromString(html, 'text/html');

    const headings = [];
    const headingTags = doc.querySelectorAll('h1, h2, h3, h4, h5, h6');

    headingTags.forEach((heading, index) => {
        headings.push({
            level: parseInt(heading.tagName[1]),
            text: heading.textContent.trim(),
            tag: heading.tagName.toLowerCase(),
            index: index,
            position: getElementPosition(heading)
        });
    });

    return headings;
}

function getElementPosition(element) {
    let position = 0;
    let current = element;

    while (current.previousElementSibling) {
        position++;
        current = current.previousElementSibling;
    }

    return position;
}

Step 2: Hierarchy Validation

function validateHierarchy(headings) {
    const issues = [];

    // Check for H1 count
    const h1Count = headings.filter(h => h.level === 1).length;

    if (h1Count === 0) {
        issues.push({
            type: 'missing_h1',
            severity: 'error',
            message: 'No H1 tag found. Every page should have exactly one H1.'
        });
    } else if (h1Count > 1) {
        issues.push({
            type: 'multiple_h1',
            severity: 'warning',
            message: `Found ${h1Count} H1 tags. Best practice is one H1 per page.`,
            headings: headings.filter(h => h.level === 1)
        });
    }

    // Check for skipped levels
    for (let i = 1; i < headings.length; i++) {
        const current = headings[i];
        const previous = headings[i - 1];

        if (current.level > previous.level + 1) {
            issues.push({
                type: 'skipped_level',
                severity: 'error',
                message: `Skipped from ${previous.tag} to ${current.tag}. Should not skip heading levels.`,
                from: previous,
                to: current
            });
        }
    }

    // Check for empty headings
    headings.forEach(heading => {
        if (heading.text.length === 0) {
            issues.push({
                type: 'empty_heading',
                severity: 'error',
                message: `Empty ${heading.tag} tag found.`,
                heading
            });
        }
    });

    // Check for very long headings
    headings.forEach(heading => {
        if (heading.level === 1 && heading.text.length > 70) {
            issues.push({
                type: 'long_h1',
                severity: 'warning',
                message: `H1 is ${heading.text.length} characters (over 70 recommended).`,
                heading
            });
        }
    });

    // Check for generic headings
    const genericTerms = ['introduction', 'more information', 'details', 'content', 'information', 'conclusion'];
    headings.forEach(heading => {
        const lowerText = heading.text.toLowerCase();
        if (genericTerms.some(term => lowerText === term || lowerText.includes(term + ':') || lowerText.includes(term + ' -'))) {
            issues.push({
                type: 'generic_heading',
                severity: 'warning',
                message: `Generic heading "${heading.text}" provides little semantic value.`,
                heading
            });
        }
    });

    return issues;
}

Step 3: Hierarchy Visualization

function visualizeHierarchy(headings) {
    let output = '';

    headings.forEach(heading => {
        const indent = '  '.repeat(heading.level - 1);
        output += `${indent}${heading.tag.toUpperCase()}: ${heading.text}\n`;
    });

    return output;
}

// Generate tree structure
function buildHeadingTree(headings) {
    const root = { children: [], level: 0 };
    const stack = [root];

    headings.forEach(heading => {
        const node = {
            ...heading,
            children: []
        };

        // Pop stack until we find parent level
        while (stack.length > 0 && stack[stack.length - 1].level >= heading.level) {
            stack.pop();
        }

        // Add to parent's children
        if (stack.length > 0) {
            stack[stack.length - 1].children.push(node);
        }

        // Push to stack
        stack.push(node);
    });

    return root.children;
}

Step 4: SEO Analysis

function analyzeSEO(headings) {
    const analysis = {
        h1Keywords: [],
        questionBasedH2s: 0,
        headingDensity: 0,
        averageHeadingLength: 0,
        recommendations: []
    };

    // Extract H1 keywords
    const h1s = headings.filter(h => h.level === 1);
    if (h1s.length === 1) {
        analysis.h1Keywords = extractKeywords(h1s[0].text);
    }

    // Count question-based H2s
    const h2s = headings.filter(h => h.level === 2);
    analysis.questionBasedH2s = h2s.filter(h => 
        h.text.match(/^(what|why|how|when|where|which|who)/i)
    ).length;

    // Calculate heading density (headings per page)
    analysis.headingDensity = headings.length;

    // Average heading length
    const totalLength = headings.reduce((sum, h) => sum + h.text.length, 0);
    analysis.averageHeadingLength = Math.round(totalLength / headings.length);

    // Generate recommendations
    if (h1s.length === 0) {
        analysis.recommendations.push('Add an H1 tag containing your primary keyword');
    } else if (h1s.length > 1) {
        analysis.recommendations.push('Consolidate to one H1 per page for clarity');
    }

    if (analysis.questionBasedH2s === 0 && h2s.length > 0) {
        analysis.recommendations.push('Consider using question-based H2s for featured snippet eligibility');
    }

    if (analysis.headingDensity < 3) {
        analysis.recommendations.push('Add more headings to improve content structure (aim for one heading per 150-300 words)');
    } else if (analysis.headingDensity > 20) {
        analysis.recommendations.push('Too many headings may dilute their value. Consolidate where appropriate.');
    }

    return analysis;
}

function extractKeywords(text) {
    // Simple keyword extraction (can be enhanced with NLP)
    return text.toLowerCase()
        .split(/\s+/)
        .filter(word => word.length > 3)
        .slice(0, 5);
}

Complete Analyzer Class

class HeadingTagsAnalyzer {
    constructor(html) {
        this.html = html;
        this.headings = extractHeadings(html);
        this.issues = validateHierarchy(this.headings);
        this.tree = buildHeadingTree(this.headings);
        this.seoAnalysis = analyzeSEO(this.headings);
    }

    getReport() {
        return {
            summary: {
                totalHeadings: this.headings.length,
                h1Count: this.headings.filter(h => h.level === 1).length,
                h2Count: this.headings.filter(h => h.level === 2).length,
                h3Count: this.headings.filter(h => h.level === 3).length,
                issueCount: this.issues.length,
                errorCount: this.issues.filter(i => i.severity === 'error').length,
                warningCount: this.issues.filter(i => i.severity === 'warning').length
            },
            headings: this.headings,
            issues: this.issues,
            tree: this.tree,
            seoAnalysis: this.seoAnalysis,
            visualization: visualizeHierarchy(this.headings)
        };
    }

    isValid() {
        return this.issues.filter(i => i.severity === 'error').length === 0;
    }
}

// Usage
const analyzer = new HeadingTagsAnalyzer(htmlContent);
const report = analyzer.getReport();

console.log('Summary:', report.summary);
console.log('Issues:', report.issues);
console.log('SEO Analysis:', report.seoAnalysis);
console.log('\nHierarchy:\n', report.visualization);

Conclusion

Building a heading tags analyzer requires HTML parsing, hierarchy validation, and SEO-focused analysis. The analyzer we've built extracts all heading tags, validates structure, and provides actionable recommendations for optimization.

Try it yourself: https://techbelievers.com/tools/heading-tags-analyzer

Building an SSL Certificate Checker: TLS Validation and Security Audit

Tech Believers — Fri, 05 Dec 2025 15:05:23 +0000

Introduction

SSL/TLS certificates are critical for web security and SEO, but they expire, get misconfigured, and develop vulnerabilities. In this guide, we'll build a comprehensive SSL certificate checker that validates certificates, checks security configuration, and monitors expiry.

Understanding SSL/TLS Certificates

SSL (Secure Sockets Layer) and its successor TLS (Transport Layer Security) encrypt data between client and server. Certificates authenticate the server's identity.

Key Components:

Subject: Domain name(s) the certificate covers
Issuer: Certificate Authority that issued it
Validity Period: Not valid before / Not valid after dates
Public Key: Used for encryption
Signature: Cryptographic proof of authenticity
Certificate Chain: Root + Intermediate certificates

Building the Checker

Step 1: Fetching Certificate Information

const tls = require('tls');
const https = require('https');

function checkSSLCertificate(hostname, port = 443) {
    return new Promise((resolve, reject) => {
        const options = {
            host: hostname,
            port: port,
            method: 'GET',
            rejectUnauthorized: false, // Allow checking invalid certs
            agent: false
        };

        const req = https.get(options, (res) => {
            const cert = res.socket.getPeerCertificate();

            if (!cert || Object.keys(cert).length === 0) {
                reject(new Error('No certificate found'));
                return;
            }

            resolve({
                subject: cert.subject,
                issuer: cert.issuer,
                validFrom: cert.valid_from,
                validTo: cert.valid_to,
                daysRemaining: getDaysRemaining(cert.valid_to),
                serialNumber: cert.serialNumber,
                fingerprint: cert.fingerprint,
                subjectAltNames: cert.subjectaltname,
                protocol: res.socket.getProtocol(),
                cipher: res.socket.getCipher()
            });
        });

        req.on('error', reject);
        req.end();
    });
}

function getDaysRemaining(validTo) {
    const expiry = new Date(validTo);
    const now = new Date();
    const diffTime = expiry - now;
    const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
    return diffDays;
}

Step 2: Certificate Validation

function validateCertificate(certInfo, hostname) {
    const validation = {
        valid: true,
        errors: [],
        warnings: []
    };

    // Check expiry
    if (certInfo.daysRemaining < 0) {
        validation.valid = false;
        validation.errors.push({
            type: 'expired',
            message: `Certificate expired ${Math.abs(certInfo.daysRemaining)} days ago`
        });
    } else if (certInfo.daysRemaining < 30) {
        validation.warnings.push({
            type: 'expiring_soon',
            message: `Certificate expires in ${certInfo.daysRemaining} days`
        });
    }

    // Check domain match
    const domainMatch = checkDomainMatch(certInfo, hostname);
    if (!domainMatch) {
        validation.valid = false;
        validation.errors.push({
            type: 'domain_mismatch',
            message: `Certificate not valid for ${hostname}`
        });
    }

    // Check protocol strength
    if (certInfo.protocol && certInfo.protocol.includes('TLSv1.0')) {
        validation.warnings.push({
            type: 'weak_protocol',
            message: 'TLS 1.0 is deprecated. Use TLS 1.2 or 1.3'
        });
    }

    // Check cipher strength
    if (certInfo.cipher && certInfo.cipher.name.includes('RC4')) {
        validation.warnings.push({
            type: 'weak_cipher',
            message: 'Weak cipher suite detected. Update server configuration'
        });
    }

    return validation;
}

function checkDomainMatch(certInfo, hostname) {
    // Check subject CN
    if (certInfo.subject.CN === hostname) {
        return true;
    }

    // Check Subject Alternative Names
    if (certInfo.subjectAltNames) {
        const altNames = certInfo.subjectAltNames.split(',').map(s => s.trim().replace('DNS:', ''));

        for (const altName of altNames) {
            if (altName === hostname) {
                return true;
            }

            // Check wildcard
            if (altName.startsWith('*.')) {
                const wildcardDomain = altName.substring(2);
                if (hostname.endsWith(wildcardDomain)) {
                    return true;
                }
            }
        }
    }

    return false;
}

Step 3: Security Audit

function auditSecurity(certInfo) {
    const audit = {
        score: 100,
        issues: []
    };

    // Protocol check
    if (certInfo.protocol) {
        if (certInfo.protocol.includes('TLSv1.3')) {
            audit.issues.push({ type: 'protocol', status: 'excellent', message: 'TLS 1.3 - Latest and most secure' });
        } else if (certInfo.protocol.includes('TLSv1.2')) {
            audit.issues.push({ type: 'protocol', status: 'good', message: 'TLS 1.2 - Secure' });
        } else if (certInfo.protocol.includes('TLSv1.1') || certInfo.protocol.includes('TLSv1.0')) {
            audit.score -= 30;
            audit.issues.push({ type: 'protocol', status: 'poor', message: 'TLS 1.0/1.1 deprecated. Upgrade to TLS 1.2+' });
        }
    }

    // Key strength check
    if (certInfo.cipher) {
        const keyBits = certInfo.cipher.bits || 0;

        if (keyBits >= 256) {
            audit.issues.push({ type: 'encryption', status: 'excellent', message: `${keyBits}-bit encryption - Very strong` });
        } else if (keyBits >= 128) {
            audit.issues.push({ type: 'encryption', status: 'good', message: `${keyBits}-bit encryption - Adequate` });
        } else {
            audit.score -= 40;
            audit.issues.push({ type: 'encryption', status: 'poor', message: `${keyBits}-bit encryption - Too weak` });
        }
    }

    // Expiry check
    if (certInfo.daysRemaining < 0) {
        audit.score = 0;
        audit.issues.push({ type: 'expiry', status: 'critical', message: 'Certificate expired!' });
    } else if (certInfo.daysRemaining < 7) {
        audit.score -= 20;
        audit.issues.push({ type: 'expiry', status: 'urgent', message: `Expires in ${certInfo.daysRemaining} days` });
    } else if (certInfo.daysRemaining < 30) {
        audit.score -= 10;
        audit.issues.push({ type: 'expiry', status: 'warning', message: `Expires in ${certInfo.daysRemaining} days - renewal recommended` });
    }

    audit.score = Math.max(0, audit.score);

    return audit;
}

Complete Checker Class

class SSLCertificateChecker {
    constructor(hostname, port = 443) {
        this.hostname = hostname;
        this.port = port;
    }

    async check() {
        try {
            const certInfo = await checkSSLCertificate(this.hostname, this.port);
            const validation = validateCertificate(certInfo, this.hostname);
            const securityAudit = auditSecurity(certInfo);

            return {
                hostname: this.hostname,
                port: this.port,
                certificate: certInfo,
                validation,
                securityAudit,
                timestamp: new Date().toISOString()
            };
        } catch (error) {
            return {
                hostname: this.hostname,
                port: this.port,
                error: error.message,
                timestamp: new Date().toISOString()
            };
        }
    }
}

// Usage
const checker = new SSLCertificateChecker('example.com');
checker.check().then(result => {
    console.log('Certificate Info:', result.certificate);
    console.log('Validation:', result.validation);
    console.log('Security Audit:', result.securityAudit);
    console.log('Security Score:', result.securityAudit.score);
});

Conclusion

Building an SSL certificate checker requires understanding TLS protocols, certificate validation, and security best practices. The checker we've built validates certificates, audits security configuration, and monitors expiry dates.

Try it yourself: https://techbelievers.com/tools/ssl-certificate-checker

Building a Broken Link Checker: Web Crawling and HTTP Status Validation

Tech Believers — Fri, 05 Dec 2025 15:02:34 +0000

Introduction

Broken links are a common problem that damages SEO, user experience, and site credibility. In this technical guide, we'll build a comprehensive broken link checker that crawls websites, validates HTTP status codes, and identifies issues.

Understanding Broken Links

A broken link returns an HTTP error status code (typically 404) when accessed. Types of broken links:

404 Not Found: Page doesn't exist
500 Internal Server Error: Server-side error
503 Service Unavailable: Server temporarily unavailable
Timeout: Server didn't respond in time
DNS Error: Domain doesn't exist

The Crawling Algorithm

Step 1: URL Normalization

function normalizeURL(url, baseURL) {
    try {
        // Handle relative URLs
        const absoluteURL = new URL(url, baseURL);

        // Remove fragment (#section)
        absoluteURL.hash = '';

        // Normalize trailing slash
        let normalized = absoluteURL.href;
        if (normalized.endsWith('/') && absoluteURL.pathname !== '/') {
            normalized = normalized.slice(0, -1);
        }

        return normalized;
    } catch (error) {
        console.error('Invalid URL:', url, error);
        return null;
    }
}

// Usage
normalizeURL('/about', 'https://example.com');
// Result: 'https://example.com/about'

normalizeURL('../contact', 'https://example.com/about/');
// Result: 'https://example.com/contact'

Step 2: Link Extraction

function extractLinks(html, baseURL) {
    const links = new Set();

    // Create DOM parser
    const parser = new DOMParser();
    const doc = parser.parseFromString(html, 'text/html');

    // Extract <a href="...">
    const anchorTags = doc.querySelectorAll('a[href]');
    anchorTags.forEach(tag => {
        const href = tag.getAttribute('href');
        const normalized = normalizeURL(href, baseURL);
        if (normalized) {
            links.add(normalized);
        }
    });

    // Extract <img src="...">
    const imgTags = doc.querySelectorAll('img[src]');
    imgTags.forEach(tag => {
        const src = tag.getAttribute('src');
        const normalized = normalizeURL(src, baseURL);
        if (normalized) {
            links.add(normalized);
        }
    });

    // Extract <link href="..."> (CSS, etc.)
    const linkTags = doc.querySelectorAll('link[href]');
    linkTags.forEach(tag => {
        const href = tag.getAttribute('href');
        const normalized = normalizeURL(href, baseURL);
        if (normalized) {
            links.add(normalized);
        }
    });

    // Extract <script src="...">
    const scriptTags = doc.querySelectorAll('script[src]');
    scriptTags.forEach(tag => {
        const src = tag.getAttribute('src');
        const normalized = normalizeURL(src, baseURL);
        if (normalized) {
            links.add(normalized);
        }
    });

    return Array.from(links);
}

Step 3: HTTP Status Checking

async function checkURL(url, timeout = 10000) {
    try {
        const controller = new AbortController();
        const timeoutId = setTimeout(() => controller.abort(), timeout);

        const response = await fetch(url, {
            method: 'HEAD', // Faster than GET
            signal: controller.signal,
            redirect: 'follow'
        });

        clearTimeout(timeoutId);

        return {
            url,
            status: response.status,
            statusText: response.statusText,
            ok: response.ok,
            redirected: response.redirected,
            finalURL: response.url,
            error: null
        };
    } catch (error) {
        if (error.name === 'AbortError') {
            return {
                url,
                status: 0,
                statusText: 'Timeout',
                ok: false,
                error: 'Request timed out'
            };
        }

        return {
            url,
            status: 0,
            statusText: 'Error',
            ok: false,
            error: error.message
        };
    }
}

Step 4: Crawling Logic

class BrokenLinkChecker {
    constructor(options = {}) {
        this.options = {
            maxDepth: options.maxDepth || 3,
            maxPages: options.maxPages || 100,
            timeout: options.timeout || 10000,
            respectRobotsTxt: options.respectRobotsTxt !== false,
            followExternal: options.followExternal || false,
            concurrency: options.concurrency || 5
        };

        this.visited = new Set();
        this.queue = [];
        this.results = [];
        this.baseURL = null;
    }

    async crawl(startURL) {
        this.baseURL = new URL(startURL).origin;
        this.queue.push({ url: startURL, depth: 0, source: null });

        while (this.queue.length > 0 && this.visited.size < this.options.maxPages) {
            // Process URLs in batches for concurrency
            const batch = this.queue.splice(0, this.options.concurrency);
            await Promise.all(batch.map(item => this.processURL(item)));
        }

        return this.results;
    }

    async processURL({ url, depth, source }) {
        // Skip if already visited
        if (this.visited.has(url)) {
            return;
        }

        this.visited.add(url);

        // Check if URL is valid
        const status = await checkURL(url, this.options.timeout);

        // Record result
        this.results.push({
            url,
            source,
            depth,
            ...status
        });

        // If URL is broken, don't crawl further
        if (!status.ok) {
            return;
        }

        // If max depth reached, don't crawl further
        if (depth >= this.options.maxDepth) {
            return;
        }

        // Only crawl HTML pages
        if (!this.isHTML(url)) {
            return;
        }

        // Fetch page content
        try {
            const response = await fetch(url);
            const html = await response.text();

            // Extract links
            const links = extractLinks(html, url);

            // Add links to queue
            for (const link of links) {
                // Skip external links unless followExternal is true
                if (!this.options.followExternal && !link.startsWith(this.baseURL)) {
                    continue;
                }

                // Skip if already visited
                if (this.visited.has(link)) {
                    continue;
                }

                // Add to queue
                this.queue.push({
                    url: link,
                    depth: depth + 1,
                    source: url
                });
            }
        } catch (error) {
            console.error('Error fetching page:', url, error);
        }
    }

    isHTML(url) {
        const ext = url.split('.').pop().split('?')[0].toLowerCase();
        const nonHTMLExtensions = ['jpg', 'jpeg', 'png', 'gif', 'pdf', 'zip', 'css', 'js'];
        return !nonHTMLExtensions.includes(ext);
    }

    getBrokenLinks() {
        return this.results.filter(result => !result.ok);
    }

    getRedirectChains() {
        return this.results.filter(result => result.redirected);
    }

    getStatistics() {
        const total = this.results.length;
        const broken = this.getBrokenLinks().length;
        const redirects = this.getRedirectChains().length;

        const statusCodes = {};
        this.results.forEach(result => {
            statusCodes[result.status] = (statusCodes[result.status] || 0) + 1;
        });

        return {
            total,
            broken,
            redirects,
            brokenPercentage: ((broken / total) * 100).toFixed(2),
            statusCodes
        };
    }
}

// Usage
const checker = new BrokenLinkChecker({
    maxDepth: 3,
    maxPages: 100,
    timeout: 10000,
    followExternal: false,
    concurrency: 5
});

const results = await checker.crawl('https://example.com');
const brokenLinks = checker.getBrokenLinks();
const stats = checker.getStatistics();

console.log('Total links checked:', stats.total);
console.log('Broken links found:', stats.broken);
console.log('Broken links:', brokenLinks);

Server-Side Implementation (Node.js)

For more robust crawling, use Node.js with libraries like axios and cheerio:

const axios = require('axios');
const cheerio = require('cheerio');
const { URL } = require('url');

class ServerSideBrokenLinkChecker {
    constructor(options = {}) {
        this.options = {
            maxDepth: options.maxDepth || 3,
            maxPages: options.maxPages || 100,
            timeout: options.timeout || 10000,
            userAgent: options.userAgent || 'Mozilla/5.0 (compatible; BrokenLinkChecker/1.0)',
            concurrency: options.concurrency || 5,
            delay: options.delay || 100 // Delay between requests (ms)
        };

        this.visited = new Set();
        this.queue = [];
        this.results = [];
        this.baseURL = null;
    }

    async crawl(startURL) {
        this.baseURL = new URL(startURL).origin;
        this.queue.push({ url: startURL, depth: 0, source: null });

        while (this.queue.length > 0 && this.visited.size < this.options.maxPages) {
            const batch = this.queue.splice(0, this.options.concurrency);
            await Promise.all(batch.map(item => this.processURL(item)));

            // Delay to avoid overwhelming the server
            await this.sleep(this.options.delay);
        }

        return this.results;
    }

    async processURL({ url, depth, source }) {
        if (this.visited.has(url)) {
            return;
        }

        this.visited.add(url);

        // Check URL status
        const status = await this.checkURL(url);

        // Record result
        this.results.push({
            url,
            source,
            depth,
            ...status
        });

        // If broken or max depth, don't crawl further
        if (!status.ok || depth >= this.options.maxDepth) {
            return;
        }

        // Only crawl HTML pages
        if (!this.isHTML(url)) {
            return;
        }

        // Fetch and parse page
        try {
            const response = await axios.get(url, {
                timeout: this.options.timeout,
                headers: {
                    'User-Agent': this.options.userAgent
                }
            });

            const $ = cheerio.load(response.data);

            // Extract links
            const links = this.extractLinks($, url);

            // Add to queue
            for (const link of links) {
                if (!this.visited.has(link) && link.startsWith(this.baseURL)) {
                    this.queue.push({
                        url: link,
                        depth: depth + 1,
                        source: url
                    });
                }
            }
        } catch (error) {
            console.error('Error fetching page:', url, error.message);
        }
    }

    async checkURL(url) {
        try {
            const response = await axios.head(url, {
                timeout: this.options.timeout,
                maxRedirects: 5,
                validateStatus: () => true, // Don't throw on any status
                headers: {
                    'User-Agent': this.options.userAgent
                }
            });

            return {
                status: response.status,
                statusText: response.statusText,
                ok: response.status >= 200 && response.status < 400,
                redirected: response.request._redirectable._redirectCount > 0,
                finalURL: response.request.res.responseUrl,
                responseTime: response.headers['x-response-time'] || null,
                error: null
            };
        } catch (error) {
            if (error.code === 'ECONNABORTED') {
                return {
                    status: 0,
                    statusText: 'Timeout',
                    ok: false,
                    error: 'Request timed out'
                };
            }

            if (error.code === 'ENOTFOUND') {
                return {
                    status: 0,
                    statusText: 'DNS Error',
                    ok: false,
                    error: 'Domain not found'
                };
            }

            return {
                status: 0,
                statusText: 'Error',
                ok: false,
                error: error.message
            };
        }
    }

    extractLinks($, baseURL) {
        const links = new Set();

        // Extract from <a href>
        $('a[href]').each((i, elem) => {
            const href = $(elem).attr('href');
            const normalized = this.normalizeURL(href, baseURL);
            if (normalized) links.add(normalized);
        });

        // Extract from <img src>
        $('img[src]').each((i, elem) => {
            const src = $(elem).attr('src');
            const normalized = this.normalizeURL(src, baseURL);
            if (normalized) links.add(normalized);
        });

        // Extract from <link href>
        $('link[href]').each((i, elem) => {
            const href = $(elem).attr('href');
            const normalized = this.normalizeURL(href, baseURL);
            if (normalized) links.add(normalized);
        });

        // Extract from <script src>
        $('script[src]').each((i, elem) => {
            const src = $(elem).attr('src');
            const normalized = this.normalizeURL(src, baseURL);
            if (normalized) links.add(normalized);
        });

        return Array.from(links);
    }

    normalizeURL(url, baseURL) {
        try {
            const absoluteURL = new URL(url, baseURL);
            absoluteURL.hash = '';
            return absoluteURL.href;
        } catch (error) {
            return null;
        }
    }

    isHTML(url) {
        const ext = url.split('.').pop().split('?')[0].toLowerCase();
        const nonHTMLExtensions = ['jpg', 'jpeg', 'png', 'gif', 'pdf', 'zip', 'css', 'js', 'svg', 'ico'];
        return !nonHTMLExtensions.includes(ext);
    }

    sleep(ms) {
        return new Promise(resolve => setTimeout(resolve, ms));
    }

    getBrokenLinks() {
        return this.results.filter(result => !result.ok);
    }

    getByStatus(status) {
        return this.results.filter(result => result.status === status);
    }

    getStatistics() {
        const total = this.results.length;
        const broken = this.getBrokenLinks().length;
        const redirects = this.results.filter(r => r.redirected).length;

        const statusCodes = {};
        this.results.forEach(result => {
            statusCodes[result.status] = (statusCodes[result.status] || 0) + 1;
        });

        return {
            total,
            broken,
            redirects,
            brokenPercentage: ((broken / total) * 100).toFixed(2),
            statusCodes,
            pagesVisited: this.visited.size
        };
    }

    exportResults(format = 'json') {
        if (format === 'json') {
            return JSON.stringify(this.results, null, 2);
        }

        if (format === 'csv') {
            const headers = ['URL', 'Source', 'Status', 'Status Text', 'Error'];
            const rows = this.results.map(r => [
                r.url,
                r.source || 'N/A',
                r.status,
                r.statusText,
                r.error || 'N/A'
            ]);

            return [headers, ...rows]
                .map(row => row.join(','))
                .join('\n');
        }

        return this.results;
    }
}

// Usage
const checker = new ServerSideBrokenLinkChecker({
    maxDepth: 3,
    maxPages: 100,
    timeout: 10000,
    concurrency: 5,
    delay: 100
});

checker.crawl('https://example.com')
    .then(() => {
        const brokenLinks = checker.getBrokenLinks();
        const stats = checker.getStatistics();

        console.log('Statistics:', stats);
        console.log('Broken Links:', brokenLinks);

        // Export to CSV
        const csv = checker.exportResults('csv');
        require('fs').writeFileSync('broken-links.csv', csv);
    })
    .catch(error => {
        console.error('Crawl failed:', error);
    });

Advanced Features

Redirect Chain Detection

function detectRedirectChains(results) {
    const chains = [];

    for (const result of results) {
        if (result.redirected && result.finalURL !== result.url) {
            chains.push({
                original: result.url,
                final: result.finalURL,
                source: result.source,
                hops: result.redirectCount || 1
            });
        }
    }

    // Find chains with multiple hops
    const multiHopChains = chains.filter(chain => chain.hops > 1);

    return {
        all: chains,
        multiHop: multiHopChains
    };
}

Robots.txt Respect

const robotsParser = require('robots-parser');

async function getRobotsTxt(baseURL) {
    try {
        const robotsURL = new URL('/robots.txt', baseURL).href;
        const response = await axios.get(robotsURL);
        return robotsParser(robotsURL, response.data);
    } catch (error) {
        return null;
    }
}

async function isAllowedByRobots(url, userAgent, robots) {
    if (!robots) return true;
    return robots.isAllowed(url, userAgent);
}

Sitemap Integration

const xml2js = require('xml2js');

async function parseSitemap(sitemapURL) {
    try {
        const response = await axios.get(sitemapURL);
        const parser = new xml2js.Parser();
        const result = await parser.parseStringPromise(response.data);

        const urls = [];

        if (result.urlset && result.urlset.url) {
            result.urlset.url.forEach(entry => {
                if (entry.loc && entry.loc[0]) {
                    urls.push(entry.loc[0]);
                }
            });
        }

        return urls;
    } catch (error) {
        console.error('Error parsing sitemap:', error);
        return [];
    }
}

Progress Tracking

class ProgressTracker {
    constructor(total) {
        this.total = total;
        this.current = 0;
        this.startTime = Date.now();
    }

    update(current) {
        this.current = current;
        const elapsed = Date.now() - this.startTime;
        const rate = current / (elapsed / 1000);
        const remaining = (this.total - current) / rate;

        console.log(`Progress: ${current}/${this.total} (${(current/this.total*100).toFixed(1)}%)`);
        console.log(`Estimated time remaining: ${Math.round(remaining)}s`);
    }
}

Best Practices

1. Rate Limiting

class RateLimiter {
    constructor(requestsPerSecond) {
        this.requestsPerSecond = requestsPerSecond;
        this.queue = [];
        this.processing = false;
    }

    async execute(fn) {
        return new Promise((resolve, reject) => {
            this.queue.push({ fn, resolve, reject });
            this.process();
        });
    }

    async process() {
        if (this.processing || this.queue.length === 0) {
            return;
        }

        this.processing = true;

        while (this.queue.length > 0) {
            const { fn, resolve, reject } = this.queue.shift();

            try {
                const result = await fn();
                resolve(result);
            } catch (error) {
                reject(error);
            }

            // Wait before next request
            await this.sleep(1000 / this.requestsPerSecond);
        }

        this.processing = false;
    }

    sleep(ms) {
        return new Promise(resolve => setTimeout(resolve, ms));
    }
}

2. Error Handling

async function safeCheckURL(url, retries = 3) {
    for (let i = 0; i < retries; i++) {
        try {
            return await checkURL(url);
        } catch (error) {
            if (i === retries - 1) {
                return {
                    url,
                    status: 0,
                    ok: false,
                    error: `Failed after ${retries} retries: ${error.message}`
                };
            }

            // Exponential backoff
            await sleep(Math.pow(2, i) * 1000);
        }
    }
}

3. Memory Management

class MemoryEfficientChecker extends BrokenLinkChecker {
    constructor(options) {
        super(options);
        this.maxResultsInMemory = options.maxResultsInMemory || 1000;
        this.resultStream = options.resultStream || null;
    }

    async processURL(item) {
        await super.processURL(item);

        // Flush results to stream if memory limit reached
        if (this.results.length >= this.maxResultsInMemory && this.resultStream) {
            await this.flushResults();
        }
    }

    async flushResults() {
        if (this.resultStream) {
            for (const result of this.results) {
                this.resultStream.write(JSON.stringify(result) + '\n');
            }
            this.results = [];
        }
    }
}

Conclusion

Building a broken link checker involves:

URL normalization and validation
Web crawling with depth and concurrency control
HTTP status code checking
Link extraction from HTML
Redirect chain detection
Performance optimization and rate limiting

The tool we've built can crawl entire websites, identify broken links, detect redirect chains, and provide detailed reports.

Try it yourself: Build your own checker or use our free tool at TechBelievers.com/tools/broken-link-checker

Building a Keyword Density Checker: Algorithm and Implementation

Tech Believers — Thu, 04 Dec 2025 19:01:46 +0000

Introduction

Keyword density analysis is a fundamental SEO tool that reveals how often specific words and phrases appear in content. In this technical guide, we'll build a keyword density checker from scratch, exploring the algorithms, data structures, and optimization techniques involved.

Understanding Keyword Density

Keyword density is calculated as:

Keyword Density = (Number of keyword occurrences / Total words) × 100

For example, if "machine learning" appears 15 times in a 1,000-word article, the keyword density is 1.5%.

But modern keyword density checkers do much more:

Analyze 1-word, 2-word, and 3-word phrases
Filter stop words
Calculate TF-IDF scores
Identify semantic variations

The Algorithm: Step-by-Step

Step 1: Text Preprocessing

function preprocessText(text) {
    // Convert to lowercase for case-insensitive analysis
    text = text.toLowerCase();

    // Remove HTML tags
    text = text.replace(/<[^>]*>/g, ' ');

    // Remove special characters but keep spaces and hyphens
    text = text.replace(/[^\w\s-]/g, ' ');

    // Normalize whitespace
    text = text.replace(/\s+/g, ' ').trim();

    return text;
}

Step 2: Tokenization

function tokenize(text) {
    // Split text into words
    const words = text.split(/\s+/);

    // Filter empty strings
    return words.filter(word => word.length > 0);
}

Step 3: Stop Word Filtering

const STOP_WORDS = new Set([
    'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
    'has', 'he', 'in', 'is', 'it', 'its', 'of', 'on', 'that', 'the',
    'to', 'was', 'will', 'with', 'the', 'this', 'but', 'they', 'have',
    'had', 'what', 'when', 'where', 'who', 'which', 'why', 'how'
]);

function filterStopWords(words, removeStopWords = true) {
    if (!removeStopWords) {
        return words;
    }

    return words.filter(word => !STOP_WORDS.has(word));
}

Step 4: N-Gram Generation

function generateNGrams(words, n) {
    const ngrams = [];

    for (let i = 0; i <= words.length - n; i++) {
        const ngram = words.slice(i, i + n).join(' ');
        ngrams.push(ngram);
    }

    return ngrams;
}

// Usage
const words = ['machine', 'learning', 'is', 'powerful'];
const bigrams = generateNGrams(words, 2);
// Result: ['machine learning', 'learning is', 'is powerful']

const trigrams = generateNGrams(words, 3);
// Result: ['machine learning is', 'learning is powerful']

Step 5: Frequency Counting

function countFrequency(items) {
    const frequency = new Map();

    for (const item of items) {
        frequency.set(item, (frequency.get(item) || 0) + 1);
    }

    return frequency;
}

Step 6: Density Calculation

function calculateDensity(frequency, totalWords) {
    const densityMap = new Map();

    for (const [word, count] of frequency) {
        const density = (count / totalWords) * 100;
        densityMap.set(word, {
            count,
            density: parseFloat(density.toFixed(2))
        });
    }

    return densityMap;
}

Step 7: Sorting and Filtering

function sortByFrequency(densityMap, minOccurrences = 2) {
    // Convert Map to array of [word, data] pairs
    const entries = Array.from(densityMap.entries());

    // Filter by minimum occurrences
    const filtered = entries.filter(([word, data]) => data.count >= minOccurrences);

    // Sort by frequency (descending)
    filtered.sort((a, b) => b[1].count - a[1].count);

    return filtered;
}

Complete Implementation

class KeywordDensityChecker {
    constructor(options = {}) {
        this.options = {
            removeStopWords: options.removeStopWords !== false,
            minWordLength: options.minWordLength || 2,
            minOccurrences: options.minOccurrences || 2,
            maxResults: options.maxResults || 50
        };

        this.stopWords = new Set([
            'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
            'has', 'he', 'in', 'is', 'it', 'its', 'of', 'on', 'that', 'the',
            'to', 'was', 'will', 'with', 'the', 'this', 'but', 'they', 'have',
            'had', 'what', 'when', 'where', 'who', 'which', 'why', 'how'
        ]);
    }

    analyze(text) {
        // Preprocess
        const processed = this.preprocessText(text);

        // Tokenize
        const allWords = this.tokenize(processed);

        // Filter stop words
        const words = this.filterStopWords(allWords);

        // Calculate total words (for density calculation)
        const totalWords = words.length;

        // Analyze 1-word keywords
        const oneWordKeywords = this.analyzeNGrams(words, 1, totalWords);

        // Analyze 2-word phrases
        const twoWordKeywords = this.analyzeNGrams(words, 2, totalWords);

        // Analyze 3-word phrases
        const threeWordKeywords = this.analyzeNGrams(words, 3, totalWords);

        return {
            totalWords: allWords.length,
            totalWordsExcludingStopWords: totalWords,
            oneWordKeywords,
            twoWordKeywords,
            threeWordKeywords,
            timestamp: new Date().toISOString()
        };
    }

    preprocessText(text) {
        text = text.toLowerCase();
        text = text.replace(/<[^>]*>/g, ' ');
        text = text.replace(/[^\w\s-]/g, ' ');
        text = text.replace(/\s+/g, ' ').trim();
        return text;
    }

    tokenize(text) {
        return text.split(/\s+/).filter(word => word.length >= this.options.minWordLength);
    }

    filterStopWords(words) {
        if (!this.options.removeStopWords) {
            return words;
        }
        return words.filter(word => !this.stopWords.has(word));
    }

    analyzeNGrams(words, n, totalWords) {
        // Generate n-grams
        const ngrams = this.generateNGrams(words, n);

        // Count frequency
        const frequency = this.countFrequency(ngrams);

        // Calculate density
        const density = this.calculateDensity(frequency, totalWords);

        // Sort and filter
        const sorted = this.sortByFrequency(density);

        // Limit results
        return sorted.slice(0, this.options.maxResults);
    }

    generateNGrams(words, n) {
        const ngrams = [];
        for (let i = 0; i <= words.length - n; i++) {
            ngrams.push(words.slice(i, i + n).join(' '));
        }
        return ngrams;
    }

    countFrequency(items) {
        const frequency = new Map();
        for (const item of items) {
            frequency.set(item, (frequency.get(item) || 0) + 1);
        }
        return frequency;
    }

    calculateDensity(frequency, totalWords) {
        const densityMap = new Map();
        for (const [word, count] of frequency) {
            if (count >= this.options.minOccurrences) {
                const density = (count / totalWords) * 100;
                densityMap.set(word, {
                    count,
                    density: parseFloat(density.toFixed(2))
                });
            }
        }
        return densityMap;
    }

    sortByFrequency(densityMap) {
        const entries = Array.from(densityMap.entries());
        entries.sort((a, b) => b[1].count - a[1].count);
        return entries.map(([word, data]) => ({ word, ...data }));
    }
}

// Usage
const checker = new KeywordDensityChecker({
    removeStopWords: true,
    minWordLength: 2,
    minOccurrences: 2,
    maxResults: 50
});

const text = `
Machine learning is a subset of artificial intelligence. 
Machine learning algorithms learn from data. 
The more data machine learning models have, the better they perform.
`;

const results = checker.analyze(text);

console.log('Total Words:', results.totalWords);
console.log('Top 1-Word Keywords:', results.oneWordKeywords.slice(0, 5));
console.log('Top 2-Word Keywords:', results.twoWordKeywords.slice(0, 5));
console.log('Top 3-Word Keywords:', results.threeWordKeywords.slice(0, 5));

Advanced Features

TF-IDF Scoring

TF-IDF (Term Frequency-Inverse Document Frequency) reveals which terms are important to a document relative to a corpus.

class TFIDFCalculator {
    constructor(documents) {
        this.documents = documents;
        this.idfCache = new Map();
        this.calculateIDF();
    }

    calculateIDF() {
        const N = this.documents.length;
        const termDocCount = new Map();

        // Count how many documents contain each term
        for (const doc of this.documents) {
            const uniqueTerms = new Set(doc.toLowerCase().split(/\s+/));
            for (const term of uniqueTerms) {
                termDocCount.set(term, (termDocCount.get(term) || 0) + 1);
            }
        }

        // Calculate IDF for each term
        for (const [term, docCount] of termDocCount) {
            const idf = Math.log(N / docCount);
            this.idfCache.set(term, idf);
        }
    }

    calculateTFIDF(document) {
        const words = document.toLowerCase().split(/\s+/);
        const totalWords = words.length;
        const termFreq = new Map();

        // Calculate TF
        for (const word of words) {
            termFreq.set(word, (termFreq.get(word) || 0) + 1);
        }

        // Calculate TF-IDF
        const tfidf = new Map();
        for (const [term, freq] of termFreq) {
            const tf = freq / totalWords;
            const idf = this.idfCache.get(term) || 0;
            tfidf.set(term, tf * idf);
        }

        // Sort by TF-IDF score
        const sorted = Array.from(tfidf.entries())
            .sort((a, b) => b[1] - a[1])
            .map(([term, score]) => ({
                term,
                score: parseFloat(score.toFixed(4))
            }));

        return sorted;
    }
}

// Usage
const documents = [
    "Machine learning is a subset of artificial intelligence",
    "Deep learning is a subset of machine learning",
    "Natural language processing uses machine learning"
];

const tfidf = new TFIDFCalculator(documents);
const scores = tfidf.calculateTFIDF(documents[0]);
console.log('TF-IDF Scores:', scores.slice(0, 5));

Keyword Prominence

Keyword prominence measures where keywords appear in the content (title, headers, body, etc.).

function calculateKeywordProminence(text, keyword) {
    const lowerText = text.toLowerCase();
    const lowerKeyword = keyword.toLowerCase();

    // Extract sections
    const title = extractTitle(text);
    const headers = extractHeaders(text);
    const firstParagraph = extractFirstParagraph(text);
    const lastParagraph = extractLastParagraph(text);

    const prominence = {
        inTitle: title.toLowerCase().includes(lowerKeyword),
        inHeaders: headers.some(h => h.toLowerCase().includes(lowerKeyword)),
        inFirstParagraph: firstParagraph.toLowerCase().includes(lowerKeyword),
        inLastParagraph: lastParagraph.toLowerCase().includes(lowerKeyword),
        totalOccurrences: (lowerText.match(new RegExp(lowerKeyword, 'g')) || []).length
    };

    // Calculate prominence score (weighted)
    let score = 0;
    if (prominence.inTitle) score += 40;
    if (prominence.inHeaders) score += 20;
    if (prominence.inFirstParagraph) score += 20;
    if (prominence.inLastParagraph) score += 10;
    if (prominence.totalOccurrences > 0) score += 10;

    prominence.score = score;

    return prominence;
}

function extractTitle(text) {
    const match = text.match(/^#\s+(.+)$/m);
    return match ? match[1] : '';
}

function extractHeaders(text) {
    const matches = text.match(/^#{2,6}\s+(.+)$/gm);
    return matches ? matches.map(m => m.replace(/^#+\s+/, '')) : [];
}

function extractFirstParagraph(text) {
    const paragraphs = text.split(/\n\n+/);
    return paragraphs[0] || '';
}

function extractLastParagraph(text) {
    const paragraphs = text.split(/\n\n+/);
    return paragraphs[paragraphs.length - 1] || '';
}

Semantic Similarity

Detect semantic variations using cosine similarity:

function cosineSimilarity(vec1, vec2) {
    let dotProduct = 0;
    let mag1 = 0;
    let mag2 = 0;

    const allKeys = new Set([...Object.keys(vec1), ...Object.keys(vec2)]);

    for (const key of allKeys) {
        const val1 = vec1[key] || 0;
        const val2 = vec2[key] || 0;

        dotProduct += val1 * val2;
        mag1 += val1 * val1;
        mag2 += val2 * val2;
    }

    mag1 = Math.sqrt(mag1);
    mag2 = Math.sqrt(mag2);

    if (mag1 === 0 || mag2 === 0) return 0;

    return dotProduct / (mag1 * mag2);
}

function findSemanticVariations(targetKeyword, allKeywords, threshold = 0.5) {
    const targetVec = createWordVector(targetKeyword);
    const variations = [];

    for (const keyword of allKeywords) {
        if (keyword === targetKeyword) continue;

        const keywordVec = createWordVector(keyword);
        const similarity = cosineSimilarity(targetVec, keywordVec);

        if (similarity >= threshold) {
            variations.push({ keyword, similarity });
        }
    }

    return variations.sort((a, b) => b.similarity - a.similarity);
}

function createWordVector(text) {
    const words = text.toLowerCase().split(/\s+/);
    const vector = {};

    for (const word of words) {
        vector[word] = (vector[word] || 0) + 1;
    }

    return vector;
}

Optimization Techniques

1. Memoization

Cache results for repeated analyses:

class CachedKeywordDensityChecker extends KeywordDensityChecker {
    constructor(options) {
        super(options);
        this.cache = new Map();
    }

    analyze(text) {
        const hash = this.hashText(text);

        if (this.cache.has(hash)) {
            return this.cache.get(hash);
        }

        const results = super.analyze(text);
        this.cache.set(hash, results);

        return results;
    }

    hashText(text) {
        // Simple hash function
        let hash = 0;
        for (let i = 0; i < text.length; i++) {
            const char = text.charCodeAt(i);
            hash = ((hash << 5) - hash) + char;
            hash = hash & hash; // Convert to 32-bit integer
        }
        return hash;
    }
}

2. Web Workers

Offload heavy processing to Web Workers:

// worker.js
self.onmessage = function(e) {
    const { text, options } = e.data;
    const checker = new KeywordDensityChecker(options);
    const results = checker.analyze(text);
    self.postMessage(results);
};

// main.js
const worker = new Worker('worker.js');

worker.onmessage = function(e) {
    const results = e.data;
    console.log('Analysis complete:', results);
};

worker.postMessage({
    text: document.body.innerText,
    options: { removeStopWords: true }
});

3. Streaming Analysis

For very large texts, use streaming:

async function* streamAnalyze(text, chunkSize = 1000) {
    const words = text.split(/\s+/);

    for (let i = 0; i < words.length; i += chunkSize) {
        const chunk = words.slice(i, i + chunkSize);
        const chunkText = chunk.join(' ');

        const checker = new KeywordDensityChecker();
        const results = checker.analyze(chunkText);

        yield {
            progress: ((i + chunk.length) / words.length) * 100,
            results
        };
    }
}

// Usage
for await (const { progress, results } of streamAnalyze(largeText)) {
    console.log(`Progress: ${progress.toFixed(2)}%`);
    console.log('Chunk results:', results);
}

Best Practices

1. Handle Edge Cases

function safeAnalyze(text) {
    // Handle empty text
    if (!text || text.trim().length === 0) {
        return {
            error: 'Text is empty',
            totalWords: 0
        };
    }

    // Handle very short text
    if (text.split(/\s+/).length < 10) {
        return {
            warning: 'Text is too short for meaningful analysis',
            totalWords: text.split(/\s+/).length
        };
    }

    // Proceed with analysis
    const checker = new KeywordDensityChecker();
    return checker.analyze(text);
}

2. Validate Input

function validateOptions(options) {
    const defaults = {
        removeStopWords: true,
        minWordLength: 2,
        minOccurrences: 2,
        maxResults: 50
    };

    const validated = { ...defaults, ...options };

    // Validate ranges
    validated.minWordLength = Math.max(1, Math.min(validated.minWordLength, 10));
    validated.minOccurrences = Math.max(1, Math.min(validated.minOccurrences, 100));
    validated.maxResults = Math.max(1, Math.min(validated.maxResults, 1000));

    return validated;
}

3. Performance Monitoring

function analyzeWithTiming(text) {
    const start = performance.now();

    const checker = new KeywordDensityChecker();
    const results = checker.analyze(text);

    const end = performance.now();
    const duration = end - start;

    return {
        ...results,
        performance: {
            duration: `${duration.toFixed(2)}ms`,
            wordsPerSecond: Math.round(results.totalWords / (duration / 1000))
        }
    };
}

Conclusion

Building a keyword density checker involves:

Text preprocessing and tokenization
N-gram generation for phrase analysis
Frequency counting and density calculation
Advanced features like TF-IDF and prominence scoring
Optimization for performance

The tool we've built can analyze content at scale, providing insights into keyword usage, semantic variations, and optimization opportunities.

Try it yourself: Build your own checker or use our free tool at TechBelievers.com/tools/keyword-density-checker

Building a Meta Tags Analyzer: Technical Deep-Dive

Tech Believers — Thu, 04 Dec 2025 19:00:14 +0000

Introduction

Meta tags are the unsung heroes of SEO. They're invisible to users but critical for search engines and social platforms. In this technical guide, we'll explore how meta tags work, why they matter, and how to build your own Meta Tags Analyzer tool.

What Are Meta Tags? (The Technical Perspective)

Meta tags are HTML elements placed in the <head> section of a document. They provide metadata about the webpage—information that describes the content but isn't displayed on the page itself.

Basic Structure

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Page Title Here</title>
    <meta name="description" content="Page description here">
    <meta name="keywords" content="keyword1, keyword2, keyword3">

    <!-- Open Graph -->
    <meta property="og:title" content="OG Title">
    <meta property="og:description" content="OG Description">
    <meta property="og:image" content="https://example.com/image.jpg">
    <meta property="og:url" content="https://example.com/page">

    <!-- Twitter Card -->
    <meta name="twitter:card" content="summary_large_image">
    <meta name="twitter:title" content="Twitter Title">
    <meta name="twitter:description" content="Twitter Description">

    <!-- Technical -->
    <link rel="canonical" href="https://example.com/page">
    <meta name="robots" content="index, follow">
</head>
<body>
    <!-- Page content -->
</body>
</html>

How Search Engines Process Meta Tags

When a search engine crawler visits your page, here's what happens:

1. HTTP Request
   - Crawler sends GET request to your URL
   - Server responds with HTML document

2. HTML Parsing
   - Crawler parses the HTML DOM
   - Extracts <head> section
   - Identifies all meta tags

3. Tag Extraction
   - Reads <title> tag
   - Reads meta name="description"
   - Reads meta property="og:*"
   - Reads link rel="canonical"
   - Reads meta name="robots"

4. Data Storage
   - Stores metadata in search index
   - Associates with page URL
   - Uses for ranking and display decisions

5. Rendering Decision
   - Determines how to display in SERPs
   - Generates search result snippet
   - Applies any robots directives

Building a Meta Tags Analyzer: The Logic

Let's build a tool that fetches a URL, extracts meta tags, and analyzes them.

Step 1: Fetch the HTML

async function fetchHTML(url) {
    try {
        const response = await fetch(url, {
            method: 'GET',
            headers: {
                'User-Agent': 'Mozilla/5.0 (compatible; MetaTagsAnalyzer/1.0)'
            }
        });

        if (!response.ok) {
            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
        }

        const html = await response.text();
        return html;
    } catch (error) {
        console.error('Error fetching URL:', error);
        throw error;
    }
}

Step 2: Parse the HTML

function parseHTML(html) {
    // Create a DOM parser
    const parser = new DOMParser();
    const doc = parser.parseFromString(html, 'text/html');

    return doc;
}

Step 3: Extract Meta Tags

function extractMetaTags(doc) {
    const metaTags = {
        basic: {},
        openGraph: {},
        twitter: {},
        technical: {}
    };

    // Extract title
    const titleTag = doc.querySelector('title');
    metaTags.basic.title = titleTag ? titleTag.textContent : null;
    metaTags.basic.titleLength = metaTags.basic.title ? metaTags.basic.title.length : 0;

    // Extract meta description
    const descTag = doc.querySelector('meta[name="description"]');
    metaTags.basic.description = descTag ? descTag.getAttribute('content') : null;
    metaTags.basic.descriptionLength = metaTags.basic.description ? metaTags.basic.description.length : 0;

    // Extract meta keywords (deprecated but still checked)
    const keywordsTag = doc.querySelector('meta[name="keywords"]');
    metaTags.basic.keywords = keywordsTag ? keywordsTag.getAttribute('content') : null;

    // Extract viewport
    const viewportTag = doc.querySelector('meta[name="viewport"]');
    metaTags.basic.viewport = viewportTag ? viewportTag.getAttribute('content') : null;

    // Extract Open Graph tags
    const ogTags = doc.querySelectorAll('meta[property^="og:"]');
    ogTags.forEach(tag => {
        const property = tag.getAttribute('property').replace('og:', '');
        metaTags.openGraph[property] = tag.getAttribute('content');
    });

    // Extract Twitter Card tags
    const twitterTags = doc.querySelectorAll('meta[name^="twitter:"]');
    twitterTags.forEach(tag => {
        const name = tag.getAttribute('name').replace('twitter:', '');
        metaTags.twitter[name] = tag.getAttribute('content');
    });

    // Extract canonical URL
    const canonicalTag = doc.querySelector('link[rel="canonical"]');
    metaTags.technical.canonical = canonicalTag ? canonicalTag.getAttribute('href') : null;

    // Extract robots directives
    const robotsTag = doc.querySelector('meta[name="robots"]');
    metaTags.technical.robots = robotsTag ? robotsTag.getAttribute('content') : null;

    // Extract language
    const htmlTag = doc.querySelector('html');
    metaTags.technical.lang = htmlTag ? htmlTag.getAttribute('lang') : null;

    // Extract structured data (JSON-LD)
    const structuredData = [];
    const jsonLdScripts = doc.querySelectorAll('script[type="application/ld+json"]');
    jsonLdScripts.forEach(script => {
        try {
            const data = JSON.parse(script.textContent);
            structuredData.push(data);
        } catch (e) {
            console.error('Error parsing JSON-LD:', e);
        }
    });
    metaTags.technical.structuredData = structuredData;

    return metaTags;
}

Step 4: Validate Meta Tags

function validateMetaTags(metaTags) {
    const issues = [];
    const warnings = [];
    const recommendations = [];

    // Validate title tag
    if (!metaTags.basic.title) {
        issues.push('Missing title tag');
    } else {
        if (metaTags.basic.titleLength < 30) {
            warnings.push('Title tag is too short (< 30 characters)');
        }
        if (metaTags.basic.titleLength > 60) {
            warnings.push('Title tag is too long (> 60 characters, will be truncated in search results)');
        }
    }

    // Validate meta description
    if (!metaTags.basic.description) {
        issues.push('Missing meta description');
    } else {
        if (metaTags.basic.descriptionLength < 70) {
            warnings.push('Meta description is too short (< 70 characters)');
        }
        if (metaTags.basic.descriptionLength > 160) {
            warnings.push('Meta description is too long (> 160 characters, will be truncated)');
        }
    }

    // Validate Open Graph
    if (!metaTags.openGraph.title) {
        warnings.push('Missing og:title (recommended for social sharing)');
    }
    if (!metaTags.openGraph.description) {
        warnings.push('Missing og:description (recommended for social sharing)');
    }
    if (!metaTags.openGraph.image) {
        warnings.push('Missing og:image (social shares will look broken)');
    }
    if (!metaTags.openGraph.url) {
        recommendations.push('Consider adding og:url for canonical social sharing');
    }

    // Validate Twitter Card
    if (!metaTags.twitter.card) {
        recommendations.push('Consider adding Twitter Card tags for better Twitter sharing');
    }

    // Validate canonical
    if (!metaTags.technical.canonical) {
        recommendations.push('Consider adding a canonical tag to avoid duplicate content issues');
    }

    // Validate viewport (mobile optimization)
    if (!metaTags.basic.viewport) {
        issues.push('Missing viewport meta tag (required for mobile optimization)');
    }

    // Validate structured data
    if (metaTags.technical.structuredData.length === 0) {
        recommendations.push('Consider adding structured data (JSON-LD) for rich snippets');
    }

    return { issues, warnings, recommendations };
}

Step 5: Generate SEO Score

function calculateSEOScore(metaTags, validation) {
    let score = 100;

    // Deduct points for issues
    score -= validation.issues.length * 15;

    // Deduct points for warnings
    score -= validation.warnings.length * 10;

    // Deduct points for missing recommendations
    score -= validation.recommendations.length * 5;

    // Bonus points for structured data
    if (metaTags.technical.structuredData.length > 0) {
        score += 10;
    }

    // Ensure score is between 0 and 100
    score = Math.max(0, Math.min(100, score));

    return score;
}

Step 6: Put It All Together

async function analyzeMetaTags(url) {
    try {
        // Fetch HTML
        console.log('Fetching URL...');
        const html = await fetchHTML(url);

        // Parse HTML
        console.log('Parsing HTML...');
        const doc = parseHTML(html);

        // Extract meta tags
        console.log('Extracting meta tags...');
        const metaTags = extractMetaTags(doc);

        // Validate
        console.log('Validating...');
        const validation = validateMetaTags(metaTags);

        // Calculate score
        const score = calculateSEOScore(metaTags, validation);

        // Return results
        return {
            url,
            metaTags,
            validation,
            score,
            timestamp: new Date().toISOString()
        };
    } catch (error) {
        console.error('Analysis failed:', error);
        throw error;
    }
}

// Usage
analyzeMetaTags('https://example.com')
    .then(results => {
        console.log('Meta Tags Analysis Results:');
        console.log('SEO Score:', results.score);
        console.log('Issues:', results.validation.issues);
        console.log('Warnings:', results.validation.warnings);
        console.log('Recommendations:', results.validation.recommendations);
    })
    .catch(error => {
        console.error('Error:', error);
    });

Advanced Features

Image Validation

For Open Graph images, validate dimensions and format:

async function validateOGImage(imageUrl) {
    try {
        const img = new Image();
        img.src = imageUrl;

        await new Promise((resolve, reject) => {
            img.onload = resolve;
            img.onerror = reject;
        });

        const validation = {
            exists: true,
            width: img.naturalWidth,
            height: img.naturalHeight,
            aspectRatio: (img.naturalWidth / img.naturalHeight).toFixed(2)
        };

        // Validate dimensions
        if (img.naturalWidth < 1200 || img.naturalHeight < 630) {
            validation.warning = 'Image is smaller than recommended (1200x630px)';
        }

        // Validate aspect ratio (should be ~1.91:1 for og:image)
        const idealRatio = 1.91;
        const actualRatio = img.naturalWidth / img.naturalHeight;
        if (Math.abs(actualRatio - idealRatio) > 0.1) {
            validation.warning = 'Image aspect ratio is not optimal for social sharing';
        }

        return validation;
    } catch (error) {
        return {
            exists: false,
            error: 'Image could not be loaded'
        };
    }
}

Structured Data Validation

Validate JSON-LD structured data:

function validateStructuredData(structuredData) {
    const results = [];

    structuredData.forEach((data, index) => {
        const validation = {
            index,
            type: data['@type'],
            valid: true,
            errors: []
        };

        // Check required @context
        if (!data['@context']) {
            validation.valid = false;
            validation.errors.push('Missing @context');
        }

        // Check required @type
        if (!data['@type']) {
            validation.valid = false;
            validation.errors.push('Missing @type');
        }

        // Type-specific validation
        if (data['@type'] === 'Article') {
            if (!data.headline) validation.errors.push('Missing headline');
            if (!data.author) validation.errors.push('Missing author');
            if (!data.datePublished) validation.errors.push('Missing datePublished');
        } else if (data['@type'] === 'Organization') {
            if (!data.name) validation.errors.push('Missing name');
            if (!data.url) validation.errors.push('Missing url');
        } else if (data['@type'] === 'Product') {
            if (!data.name) validation.errors.push('Missing name');
            if (!data.offers) validation.errors.push('Missing offers');
        }

        if (validation.errors.length > 0) {
            validation.valid = false;
        }

        results.push(validation);
    });

    return results;
}

Server-Side Implementation (Node.js)

For server-side analysis, use libraries like axios and cheerio:

const axios = require('axios');
const cheerio = require('cheerio');

async function analyzeMetaTagsServer(url) {
    try {
        // Fetch HTML
        const response = await axios.get(url, {
            headers: {
                'User-Agent': 'Mozilla/5.0 (compatible; MetaTagsAnalyzer/1.0)'
            },
            timeout: 10000
        });

        // Parse with Cheerio
        const $ = cheerio.load(response.data);

        // Extract meta tags
        const metaTags = {
            basic: {
                title: $('title').text(),
                description: $('meta[name="description"]').attr('content'),
                keywords: $('meta[name="keywords"]').attr('content'),
                viewport: $('meta[name="viewport"]').attr('content')
            },
            openGraph: {},
            twitter: {},
            technical: {
                canonical: $('link[rel="canonical"]').attr('href'),
                robots: $('meta[name="robots"]').attr('content'),
                lang: $('html').attr('lang')
            }
        };

        // Extract Open Graph
        $('meta[property^="og:"]').each((i, elem) => {
            const property = $(elem).attr('property').replace('og:', '');
            metaTags.openGraph[property] = $(elem).attr('content');
        });

        // Extract Twitter
        $('meta[name^="twitter:"]').each((i, elem) => {
            const name = $(elem).attr('name').replace('twitter:', '');
            metaTags.twitter[name] = $(elem).attr('content');
        });

        // Extract structured data
        const structuredData = [];
        $('script[type="application/ld+json"]').each((i, elem) => {
            try {
                const data = JSON.parse($(elem).html());
                structuredData.push(data);
            } catch (e) {
                console.error('Error parsing JSON-LD:', e);
            }
        });
        metaTags.technical.structuredData = structuredData;

        return metaTags;
    } catch (error) {
        console.error('Error analyzing meta tags:', error);
        throw error;
    }
}

Best Practices for Meta Tags

Title Tag

<!-- Good -->
<title>Project Management Software for Remote Teams | TechBelievers</title>

<!-- Bad -->
<title>Home</title>
<title>Welcome to Our Website | The Best Place for All Your Needs | Buy Now | Free Shipping</title>

Rules:

50-60 characters optimal
Include primary keyword naturally
Front-load important words
Include brand name (at the end)
Unique for every page

Meta Description

<!-- Good -->
<meta name="description" content="Manage remote teams effectively with our project management software. Features include task tracking, team collaboration, and real-time reporting. Try free for 14 days.">

<!-- Bad -->
<meta name="description" content="Welcome to our website.">
<meta name="description" content="project management software project management tools project management platform best project management software for remote teams">

Rules:

155-160 characters optimal
Include primary keyword naturally
Compelling value proposition
Call-to-action when appropriate
Unique for every page

Open Graph Tags

<!-- Complete Open Graph implementation -->
<meta property="og:title" content="Project Management for Remote Teams">
<meta property="og:description" content="Manage distributed teams with ease. Task tracking, collaboration, and reporting in one platform.">
<meta property="og:image" content="https://example.com/images/og-image.jpg">
<meta property="og:image:width" content="1200">
<meta property="og:image:height" content="630">
<meta property="og:url" content="https://example.com/product">
<meta property="og:type" content="website">
<meta property="og:site_name" content="TechBelievers">

Rules:

Always include og:title, og:description, og:image, og:url
Image should be 1200x630px minimum
Use high-quality, relevant images
og:title can differ from page title (optimize for social)

Conclusion

Building a Meta Tags Analyzer requires understanding HTML parsing, validation logic, and SEO best practices. The tool we've built can:

Fetch and parse any URL
Extract all meta tags
Validate against best practices
Calculate an SEO score
Provide actionable recommendations

Meta tags are the foundation of technical SEO. By analyzing and optimizing them, you can significantly improve search visibility and social sharing performance.

Try it yourself: Build your own analyzer or use our free tool at TechBelievers.com/tools/meta-tags-analyzer