import { describe, expect, it } from 'vitest';
import { tokenize } from './tokenize';

describe('docs tokenizer', () => {
	it('delegates Svelte script blocks to the TypeScript tokenizer even with attributes', () => {
		const tokens = tokenize(
			`<script lang="ts">
const label = '1 artículo';
</script>`,
			'svelte'
		);

		expect(tokens.some((token) => token.k === 'keyword' && token.t === 'const')).toBe(true);
		expect(tokens.some((token) => token.k === 'string' && token.t.includes('artículo'))).toBe(true);
	});

	it('advances over unicode identifiers in css-like content', () => {
		const tokens = tokenize('mañana: verde;', 'css');

		expect(tokens.map((token) => token.t).join('')).toBe('mañana: verde;');
	});
});