/
dim81
/
learning-spark
Обзор
Документация
Войти
/
dim81
/
learning-spark
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
src/python/PerKeyAvg.py
30 строк
817 B
Holden Karau
pep8ify the python examples
06 сен 2014, 00:24
06 сен 2014, 00:24
7ba6e22
Код
Авторство
О чём код?
""" >>> from pyspark.context import SparkContext >>> sc = SparkContext('local', 'test') >>> input = [("coffee", 1), ("pandas", 2), ("coffee", 3), ("very", 4)] >>> b = sc.parallelize(input) >>> perKeyAvg(b) """ import sys from pyspark import SparkContext def perKeyAvg(nums): """Compute the avg""" sumCount = nums.combineByKey((lambda x: (x, 1)), (lambda x, y: (x[0] + y, x[1] + 1)), (lambda x, y: (x[0] + y[0], x[1] + y[1]))) return sumCount.collectAsMap() if __name__ == "__main__": master = "local" if len(sys.argv) == 2: master = sys.argv[1] sc = SparkContext(master, "Sum") nums = sc.parallelize( [("coffee", 1), ("pandas", 2), ("coffee", 3), ("very", 4)]) avg = perKeyAvg(nums) print avg