Python-сообщество

 %%cython -a 
#cython: boundscheck=False
import numpy as np
cimport numpy as np
cpdef trivial1_cyth1_safe(text):
    cdef unsigned char[:] text_int_view = np.array(text.view(np.uint8)[::4])
    cdef int[:] c = np.zeros(256, dtype=np.int)
    cdef int l = len(text_int_view)
    cdef int i
    for i in range(l):
        c[text_int_view[i]] += 1
    return {chr(i): c[i] for i in range(len(c)) if c[i] > 0}

 %%timeit -n 5
trivial1_cyth1_safe(text)

 trivial1_cyth1_safe(text) == trivial1_cyth1(text)

 %%cython -a 
import numpy as np
cimport numpy as np
cpdef trivial1_cyth1_boundcheck(text):
    cdef int[:] text_int_view = text.view(np.int)
    cdef int[:] c = np.zeros(256, dtype=np.int)
    cdef int l = len(text_int_view )
    cdef int i
    for i in range(l):
        c[text_int_view [i]] += 1
    return {chr(i): c[i] for i in range(len(c)) if c[i] > 0}

 %%timeit -n 5
trivial1_cyth1_boundcheck(text)

 np.add.at(c, text.view(np.int), 1)

package main

import (
	"fmt"
	"infotheory/generator/utils"
	"io/ioutil"
	"math"
	"math/rand"
	"time"
)

func getRandLetter(alphabet []rune, probability []float64) rune {
	num := rand.Float64()
	for index, currentProb := range probability {
		if currentProb > num {
			return alphabet[index]
		}
	}
	return alphabet[len(probability) - 1]
}

func GenerateString(conf *utils.Config) ([]byte, map[byte]int, time.Duration) {
	start := time.Now()

	resultStr := make([]byte, conf.Length)
	counter := make(map[byte]int)
	var letter byte

	for i := int64(0); i < conf.Length; i++ {
		letter = byte(getRandLetter(conf.Alphabet, conf.Probability))
		resultStr[i] = letter
		counter[letter]++
	}

	return resultStr, counter, time.Since(start)
}

func calcEntropy(conf *utils.Config) float64 {
	var res float64
	for _, i := range conf.Probability {
		res += -i * math.Log2(i)
	}
	return res
}

func init() {
	rand.Seed(time.Now().UnixNano())
}

func main() {
	conf := utils.ConfigParser()

	text, counter, spentTime := GenerateString(&conf)
	fmt.Println("Сгенерированна строка длинной:", conf.Length)
	fmt.Println("Потрачено времени на генерацию строки и подсчет символов:", spentTime)
	fmt.Println("Энтропия равна:", calcEntropy(&conf))
	fmt.Println("Количество символов:")
	for key, value := range counter {
		fmt.Println(string(key), value)
	}

	err := ioutil.WriteFile("./out.txt", text, 0755)
	if err != nil {
		panic(err)
	}
}

package utils

import (
	"fmt"
	"github.com/olebedev/config"
	"strconv"
	"os"
	"strings"
)

type Config struct {
	Length      int64
	Alphabet    []rune
	Probability []float64
}

func parseProbability(conf *config.Config, result *Config) {
	Probability, err := conf.String("probability")
	if err != nil {
		panic(err)
	}

	prevProb := 0.0
	for _, value := range strings.Split(Probability, ",") {
		i, err := strconv.ParseFloat(value, 64)
		if err != nil{
			panic(err)
		}
		prevProb += i
		result.Probability = append(result.Probability, prevProb)
	}

	if result.Probability[len(result.Probability)-1] != 1.0 {
		panic(fmt.Sprintln("Сумма вероятностей не равна 1, проверьте конфигурацию", result.Probability))
	}
}

func ConfigParser() Config {
	conf, err := config.ParseJsonFile("/Users/alex/Golang/src/infotheory/generator/config.json")
	if err != nil {
		fmt.Println("Конфигурационный файл не найден")
		os.Exit(1)
	}

	result := new(Config)

	length, err := conf.String("length")
	if err != nil {
		panic(err)
	}

	result.Length, err = strconv.ParseInt(length, 0, 64)
	if err != nil {
		panic(err)
	}

	alphabet, err := conf.String("alphabet")
	if err != nil {
		panic(err)
	}

	for _, value := range strings.Split(alphabet, ",") {
		result.Alphabet = append(result.Alphabet, rune([]byte(value)[0]))
	}

	parseProbability(conf, result)

	return *result
}

 import numpy as np
from memory_profiler import profile
from sys import getsizeof
  
def create_text(length):
    alphabet = {str(i - 1): np.float64(i / 55.0) for i in range(1, 11)}
    return np.random.choice(list(alphabet.keys()), p=list(alphabet.values()), size=length)
  
def f_prev(text):
    return {code: count for code, count in zip(*np.unique(text, return_counts=True))}
  
@profile
def main():
    text = create_text(100000000)
    print('Size of text:', getsizeof(text))
    result = f_prev(text)
    print(result)
  
if __name__ == '__main__':
    main()

 > python -m memory_profiler alphabet.py
Size of text: 400000096
{'3': 7279434, '9': 18185304, '4': 9093148, '7': 14545076, '0': 1817613, '1': 3635281, '2': 5452784, '6': 12721588, '8': 16367275, '5': 10902497}
Filename: alphabet.py
Line #    Mem usage    Increment   Line Contents
================================================
    15     46.6 MiB      0.0 MiB   @profile
    16                             def main():
    17    426.5 MiB    379.9 MiB       text = create_text(100000000)
    18    426.8 MiB      0.3 MiB       print('Size of text:', getsizeof(text))
    19    427.3 MiB      0.5 MiB       result = f_prev(text)
    20    427.3 MiB      0.0 MiB       print(result)

 from fractions import Fraction
from itertools import repeat, chain
from array import array
from memory_profiler import profile
from sys import getsizeof
  
ALPHABET_SIZE = 10
  
def create_text(length):
    alphabet = ''.join(str(i) for i in range(ALPHABET_SIZE)).encode('ascii')
    denominator = int(ALPHABET_SIZE * (ALPHABET_SIZE + 1) / 2)
    probabilities = [Fraction(numerator, denominator) for numerator in range(1, ALPHABET_SIZE + 1)]
  
    return bytes(chain.from_iterable(repeat(sym, round(prob * length)) for sym, prob in zip(alphabet, probabilities)))
  
def count_symbols(text):
    c = array('L', repeat(0, 256))
    for symbol in text:
        c[symbol] += 1
    return {chr(i): c[i] for i in range(len(c)) if c[i] > 0}
  
@profile
def main():
    text = create_text(100000000)
    print('Size of text:', getsizeof(text))
    result = count_symbols(text)
    print(result)
  
if __name__ == '__main__':
    main()

 > python -m memory_profiler alphabet_with_array.py
Size of text: 100000033
{'9': 18181818, '1': 3636364, '4': 9090909, '8': 16363636, '0': 1818182, '3': 7272727, '6': 12727273, '5': 10909091, '2': 5454545, '7': 14545455}
Filename: alphabet_with_array.py
  
Line #    Mem usage    Increment   Line Contents
================================================
    26     37.6 MiB      0.0 MiB   @profile
    27                             def main():
    28    133.3 MiB     95.6 MiB       text = create_text(100000000)
    29    133.6 MiB      0.3 MiB       print('Size of text:', getsizeof(text))
    30    133.7 MiB      0.0 MiB       result = count_symbols(text)
    31    133.7 MiB      0.0 MiB       print(result)