Задачка на самый быстрый подсчет встречаемости слов

Input:    2.96073
Process:  0.015366
Output:   0.026757

<?php
setlocale(LC_ALL, 'C');

$text = strtolower(file_get_contents('./huge.txt'));
preg_match_all("/[a-z]+/", $text, $words);

$count = [];
foreach($words[0] as $word) if (array_key_exists($word, $count)) $count[$word]++; else $count[$word] = 1;

arsort($count, SORT_NUMERIC);
file_put_contents('./res2.txt', var_export($count, true));

~/s/ct ❯❯❯ time php8.0 count2.php

________________________________________________________
Executed in    8,86 secs   fish           external 
   usr time    7,23 secs  1273,00 micros    7,23 secs 
   sys time    1,62 secs  322,00 micros    1,62 secs 

Freundschaftsbezeigungenstadtverordnetenversammlungenfamilieneigenthümlichkeiten

➜ hyperfine 'php8.0 -dopcache.enable_cli=1 -dopcache.jit_buffer_size=128M strtr.php' 'php8.0 -dopcache.enable_cli=1 -dopcache.jit_buffer_size=128M repl.php'
Benchmark #1: php8.0 -dopcache.enable_cli=1 -dopcache.jit_buffer_size=128M strtr.php
  Time (mean ± σ):      6.368 s ±  0.056 s    [User: 4.964 s, System: 1.400 s]
  Range (min … max):    6.282 s …  6.442 s    10 runs
 
Benchmark #2: php8.0 -dopcache.enable_cli=1 -dopcache.jit_buffer_size=128M repl.php
  Time (mean ± σ):      6.585 s ±  0.126 s    [User: 5.086 s, System: 1.485 s]
  Range (min … max):    6.390 s …  6.832 s    10 runs
 
Summary
  'php8.0 -dopcache.enable_cli=1 -dopcache.jit_buffer_size=128M strtr.php' ran
    1.03 ± 0.02 times faster than 'php8.0 -dopcache.enable_cli=1 -dopcache.jit_buffer_size=128M repl.php'

➜ cat strtr.php
<?php
ini_set('memory_limit', '-1');
setlocale(LC_ALL, 'C');
$text = strtr(strtolower(file_get_contents('./huge.txt')), "-'", '  ');
$words = str_word_count($text, 1);
$count = array_count_values($words);
arsort($count, SORT_NUMERIC);
file_put_contents('./res2s.txt', var_export($count, true));

➜ cat repl.php
<?php
ini_set('memory_limit', '-1');
setlocale(LC_ALL, 'C');
$text = str_replace(['-', "'"], ' ', strtolower(file_get_contents('./huge.txt')));
$words = str_word_count($text, 1);
$count = array_count_values($words);
arsort($count, SORT_NUMERIC);
file_put_contents('./res2r.txt', var_export($count, true));

доли секунды

use hashbrown::HashMap;
use std::fs::File;
use std::io::{BufRead as _, BufReader};

fn main() {
    let file = File::open("huge.txt").expect("Can't open file");
    let mut reader = BufReader::with_capacity(1024 * 1024, file);
    let mut counters: HashMap<Vec<u8>, u64> = HashMap::new();
    let mut word = Vec::new();

    loop {
        let buf_len = {
            let buf = reader.fill_buf().expect("Reading failed");

            if buf.is_empty() {
                break;
            }

            for byte in buf.iter() {
                match byte {
                    b'a' ..= b'z' => word.push(*byte),
                    b'A' ..= b'Z' => word.push(b'a' + (*byte - b'A')),
                    _ => {
                        if !word.is_empty() {
                            if let Some(counter) = counters.get_mut(&word) {
                                *counter += 1;
                            } else {
                                counters.insert(word.clone(), 1);
                            }
                            word.clear()
                        }
                    }
                }
            }

            buf.len()
        };
        reader.consume(buf_len);
    }
    let mut counters: Vec<_> = counters
        .into_iter()
        .map(|(word, count)| (count, String::from_utf8(word).unwrap()))
        .collect();
    counters.sort_unstable_by(|l, r| r.cmp(l));
    for (count, word) in counters.iter() {
        println!("{} {}", count, word);
    }
}

hyperfine ./target/release/count-words
Benchmark 1: ./target/release/count-words
  Time (mean ± σ):      3.143 s ±  0.113 s    [User: 2.818 s, System: 0.324 s]
  Range (min … max):    2.987 s …  3.328 s    10 runs

0:01

На i7

193236060171271332087785604739148462136540850023109634354176769542513\
450459760692886729046857234760078127160930788381267275716029504619746\
769141229171070188928463568538679833185648187049077227652717176891297\
483129424668815520603687413970275896469294732531037983134647866324669\
665609938993928547290573838996165634382651623419572382344008170554615\
992557628483466631932849600120665842127659878430747184039181061674903\
601415841646896264733243115541351556257780649335455243556667314361074\
638767116850976060195890369053078165038348852431436784919273309785127\
807385782314883445831078955977629289904379383694830045313324862712905\
130392182400311887769702385900864531404180157392210451109149334352832\
955082253422147803866524648803025196834084094025745156069590766980613\
522935820520886598714639276763237791246202796497995433189057035655446\
571107863914020332507684395274334609088675457245556806048566252312307\
027270413943378675340519405729420215963379660034808227310130177180990\
452978537098693507029348913280821987354517292735252430401859759333834\
405808704771379527705534215134936452203857457762099271778804049352188\
662508987614314727511549708820437195889059606538174880968431106943454\
094257074037539145432539603152261892296167997552624170652610353164614\
183499848950047415639370700207357515615224621427027658941558327633934\
080543927617700912740834165799704976707014821959315657691202934777262\
701366897071113522700649247615988849871534811311256902478902611555961\
325734397372556642444656624963457861510805263452923338998466557599865\
859073173828488058023561540455264759779614514978001324584892148499525\
596644308753234882221867562715319176413870819519515918793312269732659\
926242088334712251258022889731874275365216609056086956268078132436195\
146010132059303035112745859358168023200069311673376829060862717435073\
016282829739195674088098325182671557703431197457408959380268500697978\
415823490669875537735421015504811413450623029586422163881489630044330\
995406342456858109402802698250564848913831643105672087479084040617225\
951290717603350384530690820905304563784517639402962909266476874495530\
448501821149863587108645884865130614187676113904739754537204423401405\
265885391359281218526396476889195210848117570970593597448229589349677\
362158000634556774077866074369615401221097864514794738449457876725782\
103073530760291688452440754492582447562813268507792002256391564008130\
640963798746130677219247329082158426961352383293566749712121723375040\
757722153320439066626755095705487175132238773137474513774985946673486\
224866703476007772048201136586336203688757246486979587964813332916874\
313326090810221795306680175738236617361557708927159642417874234658126\
830700297942501475752237437722581279570876814400646181230768518654423\
846681022137278800053812455386759478474248529217699391478186271410791\
753323555501917472860897056463068773566966783854031889978352184784166\
4834323410018255732662370891387109376

real	2m50,914s
user	2m50,558s
sys	0m0,142

Это не голословно ...

ШИБКО УМНЫЕ ...

CPU: Intel i7-3770 (8) @ 3.900GHz

time echo '2 10000000 ^ p'|dc

772192248950779024515016843066881401412217017216751421634503592305757\
----------------------------------------------------------------------
753323555501917472860897056463068773566966783854031889978352184784166\
4834323410018255732662370891387109376

real	1m50,821s
user	1m50,737s
sys	0m0,081s

#include <fstream>
#include <iostream>
#include <memory>
#include <unordered_map>
#include <string>
#include <vector>
#include <thread>
#include <stdexcept>
#include <algorithm>
using namespace std;

struct Task {
    size_t begin, end;
    unordered_map<string, size_t> res;
};

bool check_char(char *c) {
    if (*c >= 'A'  &&  *c <= 'Z')
        *c -= 'Z' - 'z';
    return *c >= 'a'  &&  *c <= 'z';
}

void make_dict(Task *task, unique_ptr<char[]> *text, size_t sz) {
    auto add = [&task, &text](size_t b, size_t e) {
        ++ task->res[string(&(*text)[b], &(*text)[e])];
    };
    size_t pos = task->begin;
    task->begin = -1;
    for (;  pos != task->end;  ++ pos)
        if (check_char(&(*text)[pos])) {
            if (task->begin == -1)
                task->begin = pos;
            if (task->end - pos == 1)
                add(task->begin, pos + 1);
        }
        else if (task->begin != -1) {
            add(task->begin, pos);
            task->begin = -1;
        }
}

int main() {
    unique_ptr<char[]> text;
    size_t sz;
    { //block
        ifstream is{"huge.txt", ios::binary | ios::ate};
        if (! is)
            throw logic_error("file was not found");
        sz = is.tellg();
        text = make_unique_for_overwrite<char[]>(sz);
        is.seekg(0);
        if(! is.read(&text[0], sz))
            throw logic_error("read error");
    }

    vector<Task> tasks;
    { //block
        if (! thread::hardware_concurrency())
            throw logic_error("unknown number of cores");
        size_t pos = 0;
        while (pos != sz) {
            tasks.push_back(Task{pos});
            pos += sz / thread::hardware_concurrency();
            for (;  pos < sz  &&  check_char(&text[pos]);  ++ pos);
            if (pos >= sz)
                pos = sz;
            else
                ++ pos;
            tasks.back().end = pos;
        }
        vector<jthread> th;
        for (Task &t : tasks)
        th.emplace_back(make_dict, &t, &text, sz);
    }

    vector<pair<string, size_t>> res;
    for (size_t i = 0;  i < tasks.size();  ++ i) {
        for (auto &el_from_cur : tasks[i].res) {
            for (size_t i_next = i+1;  i_next < tasks.size();  ++ i_next) {
                auto nh = tasks[i_next].res.extract(el_from_cur.first);
                if (nh)
                    el_from_cur.second += nh.mapped();
            }
            res.push_back(move(el_from_cur));
        }
        sort(res.begin(), res.end(), [](auto &f, auto &s) {
                    return f.second > s.second;});
    }
    for (size_t i = 0;  i < 20;  ++ i)
        cout << res[i].second << "   " << res[i].first << '\n';
}

$ g++ 1.cpp -O3 -pthread -std=c++20
$ time ./a.out 
3343241   the
1852717   and
1715705   of
1560152   to
1324244   a

real    0m7.368s
user    0m13.463s
sys     0m0.517s

real    0m1,700s
user    0m5,495s
sys     0m0,176s

real    0m1,696s
user    0m5,268s
sys     0m0,202s

real    0m4.302s
user    0m7.840s
sys     0m0.322s

main.cpp:50:16: error: ‘make_unique_for_overwrite’ was not declared in this scope
   50 |         text = make_unique_for_overwrite<char[]>(sz);
      |                ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

$ time perl -ne '$h{lc($1)}++ while (/([A-Za-z]+)/gc); END { printf "$h{$_} $_\n" for (sort {$h{$b} <=> $h{$a}} keys(%h)) }' huge.txt > out

real    0m18.404s
user    0m18.334s
sys     0m0.052s

hyperfine ./a.out
Benchmark 1: ./a.out
  Time (mean ± σ):      1.483 s ±  0.089 s    [User: 5.296 s, System: 0.413 s]
  Range (min … max):    1.380 s …  1.576 s    10 runs