buk func

.gitignore

@@ -1,4 +0,0 @@
-.Rproj.user
-.Rhistory
-.RData
-.Ruserdata

5/data science/1e/Makefile

@@ -0,0 +1,29 @@
+CC = nvcc -arch=sm_75
+DEBUG ?= false
+DIRS = dist build
+
+ifeq ($(DEBUG), false)
+    CC += -O3
+else
+    CC += -g -G
+endif
+
+.PHONY: all run
+
+all: $(DIRS) dist/app
+
+dist/app: build/main.o build/op.o
+	$(CC) $^ -o $@ -lcuda
+
+build/op.o: op.ptx
+	$(CC) $^ -dc -o $@
+
+build/main.o: main.cu
+	$(CC) $^ -ptx -o build/main.ptx
+	$(CC) $^ -rdc=true -dc -o $@
+
+$(DIRS):
+	mkdir -p $@
+
+clean:
+	rm -rf $(DIRS)

5/data science/1e/README.md

@@ -0,0 +1,23 @@
+[euclidean](https://web.archive.org/web/20230212044931/http://www-math.ucdenver.edu/~wcherowi/courses/m5410/exeucalg.html)
+[ecdsa1](https://sefiks.com/2018/02/16/elegant-signatures-with-elliptic-curve-cryptography/)
+[ecdsa2](https://learnmeabitcoin.com/technical/cryptography/elliptic-curve/ecdsa/)
+[ptx](https://philipfabianek.com/posts/cuda-ptx-introduction)
+
+высокий приоритет
+
+6, 7	State Spaces / Properties of State Spaces	Ключевое отличие от CPU! В CPU память в основном плоская (RAM, кэш). В GPU есть много типов памяти: глобальная (.global), общая для блока потоков (.shared), константная (.const), локальная (.local) и т.д. Это фундамент для написания производительного кода.
+19	Cost Estimates for Accessing State-Spaces	Прямое продолжение предыдущего пункта. Объясняет, какая память быстрая, а какая медленная. Критично для оптимизации.
+4	Operator Precedence	Синтаксис PTX похож на ассемблер, но с выражениями. Знать приоритет операторов необходимо.
+8	Fundamental Type Specifiers	Типы данных в PTX (.b8, .s16, .f32, .b64 и т.д.). Аналог byte, word, dword в x86, но с учетом специфики GPU.
+3	Predefined Identifiers	Предопределенные константы, такие как %tid, %ctaid, %ntid. Это основа модели выполнения CUDA! Вместо одного потока (RIP/EIP) у вас есть идентификаторы потока, блока и сетки.
+20	Operation Types	Классификация инструкций PTX. Поможет быстро ориентироваться в мануале.
+1	PTX Directives	Директивы ассемблера (.version, .target, .global). Аналог секций и директив в NASM (SECTION .text, global _start)
+
+средний приоритет
+
+21	Scopes	Области видимости для атомарных операций и барьеров (.cta, .cluster, .gpu, .sys). Важно для синхронизации.
+14, 40, 56	Различные таблицы про Swizzling и Layout	Касаются продвинутых техник работы с памятью и матрицами для оптимизации доступа. Актуально для low-level оптимизаций, похоже на работу с выравниванием и SIMD в x86.
+29	Summary of Floating-Point Instructions	Обзор инструкций для чисел с плавающей точкой. На GPU они крайне важны.
+30-32	Cache Operators / Eviction Priority Hints	Управление кэшем. Продвинутая тема для тонкой настройки, аналогичная prefetch-инструкциям в x86.
+53, 55, 56	Таблицы про MMA (Matrix Multiply-Accumulate)	Инструкции для тензорных ядер (аналог FMA в x86, но для матриц). Сердце производительности в AI/HPC.
+22-25	Comparison Operators	Особенности сравнений для целых и вещественных чисел (учет NaN).

5/data science/1e/main.cu

@@ -0,0 +1,161 @@
+#include <stdint.h>
+
+template <typename T, int TILE_SIZE>
+__global__ void mat_mul(T *A, T *B, T *C, int N, int M, int K) {
+    __shared__ T sA[TILE_SIZE][TILE_SIZE];
+    __shared__ T sB[TILE_SIZE][TILE_SIZE];
+    
+    int bx = blockIdx.x, by = blockIdx.y;
+    int tx = threadIdx.x, ty = threadIdx.y;
+    
+    int row = by * TILE_SIZE + ty;
+    int col = bx * TILE_SIZE + tx;
+
+    if (col >= K || row >= M) return;
+    
+    T sum = 0;
+
+    int tiles_len = (M + TILE_SIZE - 1) / TILE_SIZE;
+    
+    for (int tile = 0; tile < tiles_len; tile++) {
+        int aCol = tile * TILE_SIZE + tx;
+        int bRow = tile * TILE_SIZE + ty;
+
+        if (aCol < M) {
+            sA[ty][tx] = A[row * M + aCol];
+        } else {
+            sA[ty][tx] = 0;
+        }
+
+        sB[ty][tx] = (T)((uint64_t)B[bRow * K + col] & ((uint64_t)(bRow >= M) - 1));
+        __syncthreads();
+        
+        for (int k = 0; k < TILE_SIZE; k++) {
+            sum += sA[ty][k] * sB[k][tx];
+        }
+    }
+
+    C[row * K + col] = sum;
+}
+
+template <typename T>
+__global__ void dumb_mat_mul(T *A, T *B, T *C, int N, int M, int K) {
+    int col = blockIdx.x * blockDim.x + threadIdx.x;
+    int row = blockIdx.y * blockDim.y + threadIdx.y;
+
+    if (col >= K || row >= M) return;
+
+    T sum = 0;
+    for (int i = 0; i < M; i++) {
+        sum += A[row * M + i] * B[i * K + col];
+    }
+    C[row * K + col] = sum;
+}
+
+#define N 1024
+#define M 1024
+#define K 1024
+#define NO_PRINT 1
+#define GRID_DIM 1
+#define BLOCK_DIM 32
+
+#define MAT_TYPE int
+#define MAT_FMT "%d\t"
+#define A_LEN (N * M)
+#define B_LEN (M * K)
+#define C_LEN (N * K)
+#define A_SIZE (sizeof(MAT_TYPE) * N * M)
+#define B_SIZE (sizeof(MAT_TYPE) * M * K)
+#define C_SIZE (sizeof(MAT_TYPE) * N * K)
+
+#include <cstdio>
+#include <random>
+#include <chrono>
+using namespace std::chrono;
+
+template <typename T>
+void mat_print(T *a, const char *fmt, int n, int m) {
+    for (auto row = 0; row < n; row++) {
+        for (auto col = 0; col < m; col++) {
+            printf(fmt, a[row * m + col]);
+        }
+        printf("\n");
+    }
+}
+
+int main() {
+    std::random_device rd;
+    std::mt19937 engine(rd());
+    std::uniform_int_distribution<MAT_TYPE> dist(1, 10);
+
+    auto buf = (MAT_TYPE *)malloc(A_SIZE + B_SIZE + C_SIZE);
+    for (auto i = 0; i < A_LEN + B_LEN; i++) {
+        buf[i] = dist(engine);
+    }
+
+    MAT_TYPE *a = buf;
+    MAT_TYPE *b = a + A_LEN;
+    MAT_TYPE *c = b + B_LEN;
+
+#if NO_PRINT==0
+    printf("\na\n");
+    mat_print(a, MAT_FMT, N, M);
+    printf("\nb\n");
+    mat_print(b, MAT_FMT, M, K);
+#endif
+
+    MAT_TYPE *d_a, *d_b, *d_c;
+    cudaMalloc(&d_a, A_SIZE);
+    cudaMalloc(&d_b, B_SIZE);
+    cudaMalloc(&d_c, C_SIZE);
+
+    cudaMemcpy(d_a, a, A_SIZE, cudaMemcpyHostToDevice);
+    cudaMemcpy(d_b, b, B_SIZE, cudaMemcpyHostToDevice);
+
+    dim3 gridDim(GRID_DIM, GRID_DIM);
+    dim3 blockDim(BLOCK_DIM, BLOCK_DIM);
+
+    int cycles = 0;
+    microseconds duration(0);
+
+    while (duration.count() < 1e6) {
+        auto start = high_resolution_clock::now();
+        mat_mul<MAT_TYPE, BLOCK_DIM><<<gridDim, blockDim>>>(d_a, d_b, d_c, N, M, K);
+        cudaDeviceSynchronize();
+        auto end = high_resolution_clock::now();
+
+        cycles++;
+        duration += duration_cast<microseconds>(end - start);
+    }
+
+#if NO_PRINT==0
+    cudaMemcpy(c, d_c, C_SIZE, cudaMemcpyDeviceToHost);
+    printf("\nc\n");
+    mat_print(c, MAT_FMT, N, K);
+#endif
+    printf("optimized mul take %f usec avg in %d cycles\n", (float)(duration.count()) / cycles, cycles);
+
+    cycles = 0;
+    duration = microseconds(0);
+    while (duration.count() < 1e6) {
+        auto start = high_resolution_clock::now();
+        dumb_mat_mul<MAT_TYPE><<<gridDim, blockDim>>>(d_a, d_b, d_c, N, M, K);
+        cudaDeviceSynchronize();
+        auto end = high_resolution_clock::now();
+
+        cycles++;
+        duration += duration_cast<microseconds>(end - start);
+    }
+
+#if NO_PRINT==0
+    cudaMemcpy(c, d_c, C_SIZE, cudaMemcpyDeviceToHost);
+    printf("\nc\n");
+    mat_print(c, MAT_FMT, N, K);
+#endif
+    printf("dumb mul take %f usec avg in %d cycles\n", (float)(duration.count()) / cycles, cycles);
+
+    cudaFree(a);
+    cudaFree(b);
+    cudaFree(c);
+    free(buf);
+}

5/data science/1e/main.py

@@ -0,0 +1,27 @@
+import sys, time, math
+import numpy as np
+import cupy as cp
+
+def measure(a, b):
+    duration = 0
+    cycles = 0
+    while (duration < 1):
+        start = time.perf_counter()
+        c = a @ b
+        cp.cuda.Stream.null.synchronize()
+        end = time.perf_counter()
+
+        duration += end - start
+        cycles += 1
+        
+    return duration / cycles
+
+n = 1024
+
+a = np.random.rand(n, n).astype(np.float32)
+b = np.random.rand(n, n).astype(np.float32)
+print('numpy take', measure(a, b) * 1e6, 'usec')
+
+a = cp.random.rand(n, n, dtype = cp.float32)
+b = cp.random.rand(n, n, dtype = cp.float32)
+print('cupy take', measure(a, b) * 1e6, 'usec')

5/data science/1e/op.ptx

@@ -0,0 +1,171 @@
+.version 8.4
+.target sm_75
+.address_size 64
+
+.visible .func add_u16(
+    .param .b64 out_c,
+    .param .align 16 .b8 in_a[16],
+    .param .align 16 .b8 in_b[16]
+) {
+    .reg .u64 %ra<2>, %rb<2>;
+    .reg .b64 %rdc;
+    
+    ld.param.b64 %rdc, [out_c];
+    
+    ld.param.v2.u64 {%ra1, %ra0}, [in_a];
+    ld.param.v2.u64 {%rb1, %rb0}, [in_b];
+
+    add.cc.u64 %ra0, %ra0, %rb0;
+    addc.u64 %ra1, %ra1, %rb1;
+
+    st.v2.u64 [%rdc], {%ra1, %ra0};
+
+    ret;
+}
+
+.visible .func sub_u16(
+    .param .b64 out_c,
+    .param .align 16 .b8 in_a[16],
+    .param .align 16 .b8 in_b[16]
+) {
+    .reg .u64 %ra<2>, %rb<2>;
+    .reg .b64 %rdc;
+    
+    ld.param.b64 %rdc, [out_c];
+    
+    ld.param.v2.u64 {%ra1, %ra0}, [in_a];
+    ld.param.v2.u64 {%rb1, %rb0}, [in_b];
+
+    sub.cc.u64 %ra0, %ra0, %rb0;
+    subc.u64 %ra1, %ra1, %rb1;
+
+    st.v2.u64 [%rdc], {%ra1, %ra0};
+
+    ret;
+}
+
+.visible .func add_u32(
+    .param .b64 out_c,
+    .param .align 16 .b8 in_a[32],
+    .param .align 16 .b8 in_b[32]
+) {
+    .reg .u64 %ra<4>, %rb<4>;
+    .reg .b64 %rdc;
+    
+    ld.param.b64 %rdc, [out_c];
+    
+    ld.param.v2.u64 {%ra3, %ra2}, [in_a];
+    ld.param.v2.u64 {%ra1, %ra0}, [in_a + 16];
+    ld.param.v2.u64 {%rb3, %rb2}, [in_b];
+    ld.param.v2.u64 {%rb1, %rb0}, [in_b + 16];
+
+    add.cc.u64 %ra0, %ra0, %rb0;
+    addc.cc.u64 %ra1, %ra1, %rb1;
+    addc.cc.u64 %ra2, %ra2, %rb2;
+    addc.u64 %ra3, %ra3, %rb3;
+
+    st.v2.u64 [%rdc], {%ra3, %ra2};
+    st.v2.u64 [%rdc + 16], {%ra1, %ra0};
+
+    ret;
+}
+
+.visible .func sub_u32(
+    .param .b64 out_c,
+    .param .align 16 .b8 in_a[32],
+    .param .align 16 .b8 in_b[32]
+) {
+    .reg .u64 %ra<4>, %rb<4>;
+    .reg .b64 %rdc;
+    
+    ld.param.b64 %rdc, [out_c];
+    
+    ld.param.v2.u64 {%ra3, %ra2}, [in_a];
+    ld.param.v2.u64 {%ra1, %ra0}, [in_a + 16];
+    ld.param.v2.u64 {%rb3, %rb2}, [in_b];
+    ld.param.v2.u64 {%rb1, %rb0}, [in_b + 16];
+
+    sub.cc.u64 %ra0, %ra0, %rb0;
+    subc.cc.u64 %ra1, %ra1, %rb1;
+    subc.cc.u64 %ra2, %ra2, %rb2;
+    subc.u64 %ra3, %ra3, %rb3;
+
+    st.v2.u64 [%rdc], {%ra3, %ra2};
+    st.v2.u64 [%rdc + 16], {%ra1, %ra0};
+
+    ret;
+}
+
+.visible .func mul_lo_u16(
+    .param .b64 out_c,
+    .param .align 16 .b8 in_a[16],
+    .param .align 16 .b8 in_b[16]
+) {
+    .reg .u64 %a, %b, %c, %d, %a_b, %c_d;
+    .reg .u64 %ac, %bd_hi, %bd_lo, %p;
+    .reg .b64 %rdc;
+
+    ld.param.b64 %rdc, [out_c];
+
+    ld.param.v2.u64 {%a, %b}, [in_a];
+    ld.param.v2.u64 {%c, %d}, [in_b];
+
+    mul.lo.u64 %ac, %a, %c;
+    mul.lo.u64 %bd_lo, %b, %d;
+    mul.hi.u64 %bd_hi, %b, %d;
+
+    add.u64 %a_b, %a, %b;
+    add.u64 %c_d, %c, %d;
+
+    mul.lo.u64 %p, %a_b, %c_d;
+    
+    sub.u64 %p, %p, %ac;
+    sub.u64 %p, %p, %bd_lo;
+
+    add.u64 %p, %p, %bd_hi;
+
+    st.v2.u64 [%rdc], {%p, %bd_lo};
+    
+    ret;
+}
+
+.visible .func mul_u16(
+    .param .b64 out_c_hi,
+    .param .b64 out_c_lo,
+    .param .align 16 .b8 in_a[16],
+    .param .align 16 .b8 in_b[16]
+) {
+  .reg .u64 %a, %b, %c, %d;
+  .reg .u64 %a_b_hi, %a_b_lo, %c_d_hi, %c_d_lo;
+  .reg .u64 %p_hi, %p_lo, %p_hi2, %p_lo2;
+  .reg .u64 %ac_hi, %ac_lo, %bd_hi, %bd_lo;
+  .reg .b64 %rdc_hi, %rdc_lo;
+
+  ld.param.b64 %rdc_hi, [out_c_hi];
+  ld.param.b64 %rdc_lo, [out_c_lo];
+
+  ld.param.v2.u64 {%a, %b}, [in_a];
+  ld.param.v2.u64 {%c, %d}, [in_b];
+
+  mul.lo.u64 %ac_lo, %a, %c;
+  mul.hi.u64 %ac_hi, %a, %c;
+  mul.lo.u64 %bd_lo, %b, %d;
+  mul.hi.u64 %bd_hi, %b, %d;
+
+  add.cc.u64 %a_b_lo, %a, %b;
+  addc.u64 %a_b_hi, %a, %b;
+  add.cc.u64 %c_d_lo, %c, %d;
+  addc.u64 %c_d_hi, %c, %d;
+
+  mul.lo.u64 %p_lo, %a_b_lo, %c_d_lo;
+  mul.hi.u64 %p_hi, %a_b_lo, %c_d_lo;
+  mul.lo.u64 %p_hi2, %a_b_hi, %c_d_hi;
+  
+
+
+
+  st.v2.u64 [%rdc_lo], {%p_hi, %p_lo};
+  st.v2.u64 [%rdc_hi], {%a_b_lo, %p_hi2};
+
+  ret;
+}

5/data science/1e/secp256k1.cu

@@ -0,0 +1,205 @@
+#include <stdio.h>
+#include <stdint.h>
+
+extern "C" __device__ void add_u16(
+    ulonglong2 *out_c,
+    ulonglong2 in_a,
+    ulonglong2 in_b
+);
+
+extern "C" __device__ void sub_u16(
+    ulonglong2 *out_c,
+    ulonglong2 in_a,
+    ulonglong2 in_b
+);
+
+extern "C" __device__ void add_u32(
+    ulonglong4 *out_c,
+    ulonglong4 in_a,
+    ulonglong4 in_b
+);
+
+extern "C" __device__ void sub_u32(
+    ulonglong4 *out_c,
+    ulonglong4 in_a,
+    ulonglong4 in_b
+);
+
+extern "C" __device__ void mul_lo_u16(
+    ulonglong2 *out_c,
+    ulonglong2 in_a,
+    ulonglong2 in_b
+);
+
+extern "C" __device__ void mul_u16(
+    ulonglong2 *out_c_hi,
+    ulonglong2 *out_c_lo,
+    ulonglong2 in_a,
+    ulonglong2 in_b
+);
+
+__device__ bool equ_u16(ulonglong2 a, ulonglong2 b) {
+    return a.x == b.x && a.y == b.y;
+}
+
+__device__ bool equ_u32(ulonglong4 a, ulonglong4 b) {
+    return a.x == b.x &&
+        a.y == b.y &&
+        a.z == b.z &&
+        a.w == b.w;
+}
+
+__device__ int cmp_u32(ulonglong4 a, ulonglong4 b) {
+  if (a.x < b.x)
+    return -1;
+  else if (a.x > b.x)
+    return 1;
+  
+  if (a.y < b.y)
+    return -1;
+  else if (a.y > b.y)
+    return 1;
+
+  if (a.z < b.z)
+    return -1;
+  else if (a.z > b.z)
+    return 1;
+
+  if (a.w < b.w)
+    return -1;
+  else if (a.w > b.w)
+    return 1;
+
+  return 0;
+}
+
+__device__ void mul_lo_u32(
+    ulonglong4 *out_c,
+    ulonglong4 in_a,
+    ulonglong4 in_b
+) {
+    auto a = (ulonglong2 *)&in_a.x;
+    auto b = (ulonglong2 *)&in_a.z;
+    auto c = (ulonglong2 *)&in_b.x;
+    auto d = (ulonglong2 *)&in_b.z;
+    ulonglong2 a_b, c_d, ac, bd_hi, bd_lo, p;
+
+    mul_lo_u16(&ac, *a, *c);
+    mul_u16(&bd_hi, &bd_lo, *b, *d);
+
+    add_u16(&a_b, *a, *b);
+    add_u16(&c_d, *c, *d);
+
+    mul_lo_u16(&p, a_b, c_d);
+
+    sub_u16(&p, p, ac);
+    sub_u16(&p, p, bd_lo);
+    add_u16(&p, p, bd_hi);
+
+    out_c->x = p.x;
+    out_c->y = p.y;
+    out_c->z = bd_lo.x;
+    out_c->w = bd_lo.y;
+}
+
+__device__ void print_u16(ulonglong2 a) {
+    printf("0x%016llx.%016llx\n", a.x, a.y);
+}
+
+__device__ void print_u32(ulonglong4 a) {
+    printf("0x%016llx.%016llx.%016llx.%016llx\n", a.x, a.y, a.z, a.w);
+}
+
+#define U8_MAX 0xFFFFFFFFFFFFFFFF
+#define U16_MAX {U8_MAX, U8_MAX}
+#define U32_MAX {U8_MAX, U8_MAX, U8_MAX, U8_MAX}
+
+__global__ void test(bool *passed) {
+    *passed = true;
+    {
+        ulonglong4 a = U32_MAX;
+        ulonglong4 b = {0, 0, 0, 1};
+        ulonglong4 c = {0, 0, 0, 0};
+        add_u32(&a, a, b);
+        if (!equ_u32(a, c)) {
+            printf("add_u32\n");
+            print_u32(a);
+            *passed = false;
+        }
+    }
+    {
+        ulonglong4 a = {0, 0, 0, 0};
+        ulonglong4 b = {0, 0, 0, 1};
+        ulonglong4 c = U32_MAX;
+        sub_u32(&a, a, b);
+        if (!equ_u32(a, c)) {
+            printf("sub_u32\n");
+            print_u32(a);
+            *passed = false;
+        }
+    }
+    {
+        ulonglong2 a = U16_MAX;
+        ulonglong2 b = {0, U8_MAX};
+        ulonglong2 c = {U8_MAX, 1};
+        mul_lo_u16(&a, a, b);
+        if (!equ_u16(a, c)) {
+            printf("mul_lo_u16\n");
+            print_u16(a);
+            *passed = false;
+        }
+    }
+    {
+        ulonglong2 a = U16_MAX;
+        ulonglong2 b = {0, U8_MAX};
+        ulonglong2 c_hi = {0, U8_MAX - 1};
+        ulonglong2 c_lo = {U8_MAX, 1};
+        mul_u16(&a, &b, a, b);
+        if (!equ_u16(a, c_hi) || !equ_u16(b, c_lo)) {
+            printf("mul_u16\n");
+            print_u16(a);
+            print_u16(b);
+            *passed = false;
+        }
+        a = U16_MAX;
+        b = U16_MAX;
+        c_hi = {U8_MAX, U8_MAX - 1};
+        c_lo = {0, 1};
+        mul_u16(&a, &b, a, b);
+        if (!equ_u16(a, c_hi) || !equ_u16(b, c_lo)) {
+            printf("mul_u16\n");
+            print_u16(a);
+            print_u16(b);
+            *passed = false;
+        }
+    }
+    {
+        ulonglong4 a = U32_MAX;
+        ulonglong4 b = {0, 0, U8_MAX, U8_MAX};
+        ulonglong4 c = {U8_MAX, U8_MAX, 0, 1};
+        mul_lo_u32(&a, a, b);
+        if (!equ_u32(a, c)) {
+            printf("mul_lo_u32\n");
+            print_u32(a);
+            *passed = false;
+        }
+    }
+}
+
+int main() {
+    bool test_passed, *d_test_passed;
+    cudaMalloc(&d_test_passed, sizeof(bool));
+	
+    test<<<1, 1>>>(d_test_passed);
+    cudaDeviceSynchronize();
+
+    cudaMemcpy(&test_passed, d_test_passed, sizeof(bool), cudaMemcpyDeviceToHost);
+    cudaFree(d_test_passed);
+
+    if (!test_passed) {
+        printf("test not passed\n");
+        return 1;
+    }
+
+	return 0;
+}

5/data science/1e/test.py

@@ -0,0 +1,22 @@
+U8_MAX = 0xFFFFFFFFFFFFFFFF
+U16_MAX = U8_MAX << 64 | U8_MAX
+U32_MAX = U16_MAX << 128 | U16_MAX
+
+
+def dothex(num):
+    strhex = hex(num)[2:]
+    dothex = strhex[-16:]
+    strhex = strhex[:-16]
+
+    while len(strhex) > 0:
+        dothex = strhex[-16:] + '.' + dothex
+        strhex = strhex[:-16]
+
+    return '0x' + dothex
+
+print('mul_u16', dothex((U16_MAX * U8_MAX >> 128) %
+      (U16_MAX + 1)), dothex(U16_MAX * U8_MAX % (U16_MAX + 1)))
+print('mul_u16', dothex((U16_MAX * U16_MAX >> 128) %
+      (U16_MAX + 1)), dothex(U16_MAX * U16_MAX % (U16_MAX + 1)))
+print('mul_lo_u32', dothex(U32_MAX * U16_MAX % (U32_MAX + 1)))
+print('div_lo_u32', dothex(U32_MAX // U8_MAX), dothex(U32_MAX - U32_MAX // U8_MAX))

5/data science/2/3-03_dbscan.ipynb

@@ -227,7 +227,7 @@
     }
    ],
    "source": [
-    "dbscan = cuml.DBSCAN(eps=5000)eps=5000)\n",
+    "dbscan = cuml.DBSCAN(eps=5000)\n",
     "# dbscan = cuml.DBSCAN(eps=10000)\n",
     "\n",
     "infected_df = gdf[gdf['infected'] == 1].reset_index()\n",

5/data science/2/3-06_xgboost.ipynb

@@ -438,6 +438,13 @@
     "Now we are ready to train the model."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": 40,
@@ -663,7 +670,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 48,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {

5/data science/2/3-07_triton.ipynb

@@ -120,7 +120,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "61d898fb-a8d2-4d1c-a13f-2c4be6c18969",
    "metadata": {},
    "outputs": [],
@@ -154,7 +154,7 @@
     "  {{\n",
     "    key: \"storage_type\"\n",
     "    value: {{ string_value: \"AUTO\" }}\n",
-    "  }}\n",
+    "  }}т\n",
     "]\n",
     "\n",
     "dynamic_batching {{\n",

5/data science/2/4-02_find_infected.ipynb

@@ -0,0 +1,818 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<a href=\"https://www.nvidia.com/dli\"><img src=\"images/DLI_Header.png\" alt=\"Header\" style=\"width: 400px;\"/></a>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Week 1: Find Clusters of Infected People"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<span style=\"color:red\">\n",
+    "**URGENT WARNING**\n",
+    "\n",
+    "We have been receiving reports from health facilities that a new, fast-spreading virus has been discovered in the population. To prepare our response, we need to understand the geospatial distribution of those who have been infected. Find out whether there are identifiable clusters of infected individuals and where they are.    \n",
+    "</span>\n",
+    "\n",
+    "Your goal for this notebook will be to estimate the location of dense geographic clusters of infected people using incoming data from week 1 of the simulated epidemic."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 65,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The cudf.pandas extension is already loaded. To reload it, use:\n",
+      "  %reload_ext cudf.pandas\n"
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext cudf.pandas\n",
+    "import pandas as pd\n",
+    "import cuml\n",
+    "\n",
+    "import cupy as cp"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load Data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Begin by loading the data you've received about week 1 of the outbreak into a cuDF-accelerated pandas DataFrame. The data is located at `'./data/week1.csv'`. For this notebook you will only need the `'lat'`, `'long'`, and `'infected'` columns. Either drop the columns after loading, or use the `pd.read_csv` named argument `usecols` to provide a list of only the columns you need."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 66,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>lat</th>\n",
+       "      <th>long</th>\n",
+       "      <th>infected</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>54.522511</td>\n",
+       "      <td>-1.571896</td>\n",
+       "      <td>False</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>54.554031</td>\n",
+       "      <td>-1.524968</td>\n",
+       "      <td>False</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>54.552483</td>\n",
+       "      <td>-1.435203</td>\n",
+       "      <td>False</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>54.537186</td>\n",
+       "      <td>-1.566215</td>\n",
+       "      <td>False</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>54.528210</td>\n",
+       "      <td>-1.588462</td>\n",
+       "      <td>False</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "         lat      long infected\n",
+       "0  54.522511 -1.571896    False\n",
+       "1  54.554031 -1.524968    False\n",
+       "2  54.552483 -1.435203    False\n",
+       "3  54.537186 -1.566215    False\n",
+       "4  54.528210 -1.588462    False"
+      ]
+     },
+     "execution_count": 66,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df = pd.read_csv('./data/week1.csv', dtype = {\n",
+    "    'lat': 'float32',\n",
+    "    'long': 'float32',\n",
+    "    'infected': 'category',\n",
+    "}, usecols = ['lat', 'long', 'infected'])\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Make Data Frame of the Infected"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Make a new DataFrame `infected_df` that contains only the infected members of the population.\n",
+    "\n",
+    "**Tip**: Reset the index of `infected_df` with `.reset_index(drop=True)`. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[28928759 28930512 28930904 ... 57410428 57411005 57411919]\n",
+      "[    0     1     2 ... 18145 18146 18147]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>lat</th>\n",
+       "      <th>long</th>\n",
+       "      <th>infected</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>54.472767</td>\n",
+       "      <td>-1.654932</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>54.529720</td>\n",
+       "      <td>-1.667143</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>54.512981</td>\n",
+       "      <td>-1.589866</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>54.522320</td>\n",
+       "      <td>-1.380694</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>54.541656</td>\n",
+       "      <td>-1.613490</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "         lat      long infected\n",
+       "0  54.472767 -1.654932     True\n",
+       "1  54.529720 -1.667143     True\n",
+       "2  54.512981 -1.589866     True\n",
+       "3  54.522320 -1.380694     True\n",
+       "4  54.541656 -1.613490     True"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "infected_df = df[df['infected'] == True]\n",
+    "print(infected_df.index.values)\n",
+    "\n",
+    "infected_df = infected_df.reset_index(drop=True)\n",
+    "\n",
+    "print(infected_df.index.values)\n",
+    "infected_df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Make Grid Coordinates for Infected Locations"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Provided for you in the next cell (which you can expand by clicking on the \"...\" and contract again after executing by clicking on the blue left border of the cell) is the lat/long to OSGB36 grid coordinates converter you used earlier in the workshop. Use this converter to create grid coordinate values stored in `northing` and `easting` columns of the `infected_df` you created in the last step."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# https://www.ordnancesurvey.co.uk/docs/support/guide-coordinate-systems-great-britain.pdf\n",
+    "\n",
+    "def latlong2osgbgrid_cupy(lat, long, input_degrees=True):\n",
+    "    '''\n",
+    "    Converts latitude and longitude (ellipsoidal) coordinates into northing and easting (grid) coordinates, using a Transverse Mercator projection.\n",
+    "    \n",
+    "    Inputs:\n",
+    "    lat: latitude coordinate (N)\n",
+    "    long: longitude coordinate (E)\n",
+    "    input_degrees: if True (default), interprets the coordinates as degrees; otherwise, interprets coordinates as radians\n",
+    "    \n",
+    "    Output:\n",
+    "    (northing, easting)\n",
+    "    '''\n",
+    "    \n",
+    "    if input_degrees:\n",
+    "        lat = lat * cp.pi/180\n",
+    "        long = long * cp.pi/180\n",
+    "\n",
+    "    a = 6377563.396\n",
+    "    b = 6356256.909\n",
+    "    e2 = (a**2 - b**2) / a**2\n",
+    "\n",
+    "    N0 = -100000 # northing of true origin\n",
+    "    E0 = 400000 # easting of true origin\n",
+    "    F0 = .9996012717 # scale factor on central meridian\n",
+    "    phi0 = 49 * cp.pi / 180 # latitude of true origin\n",
+    "    lambda0 = -2 * cp.pi / 180 # longitude of true origin and central meridian\n",
+    "    \n",
+    "    sinlat = cp.sin(lat)\n",
+    "    coslat = cp.cos(lat)\n",
+    "    tanlat = cp.tan(lat)\n",
+    "    \n",
+    "    latdiff = lat-phi0\n",
+    "    longdiff = long-lambda0\n",
+    "\n",
+    "    n = (a-b) / (a+b)\n",
+    "    nu = a * F0 * (1 - e2 * sinlat ** 2) ** -.5\n",
+    "    rho = a * F0 * (1 - e2) * (1 - e2 * sinlat ** 2) ** -1.5\n",
+    "    eta2 = nu / rho - 1\n",
+    "    M = b * F0 * ((1 + n + 5/4 * (n**2 + n**3)) * latdiff - \n",
+    "                  (3*(n+n**2) + 21/8 * n**3) * cp.sin(latdiff) * cp.cos(lat+phi0) +\n",
+    "                  15/8 * (n**2 + n**3) * cp.sin(2*(latdiff)) * cp.cos(2*(lat+phi0)) - \n",
+    "                  35/24 * n**3 * cp.sin(3*(latdiff)) * cp.cos(3*(lat+phi0)))\n",
+    "    I = M + N0\n",
+    "    II = nu/2 * sinlat * coslat\n",
+    "    III = nu/24 * sinlat * coslat ** 3 * (5 - tanlat ** 2 + 9 * eta2)\n",
+    "    IIIA = nu/720 * sinlat * coslat ** 5 * (61-58 * tanlat**2 + tanlat**4)\n",
+    "    IV = nu * coslat\n",
+    "    V = nu / 6 * coslat**3 * (nu/rho - cp.tan(lat)**2)\n",
+    "    VI = nu / 120 * coslat ** 5 * (5 - 18 * tanlat**2 + tanlat**4 + 14 * eta2 - 58 * tanlat**2 * eta2)\n",
+    "\n",
+    "    northing = I + II * longdiff**2 + III * longdiff**4 + IIIA * longdiff**6\n",
+    "    easting = E0 + IV * longdiff + V * longdiff**3 + VI * longdiff**5\n",
+    "\n",
+    "    return(northing, easting)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 69,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>lat</th>\n",
+       "      <th>long</th>\n",
+       "      <th>infected</th>\n",
+       "      <th>northing</th>\n",
+       "      <th>easting</th>\n",
+       "      <th>cluster</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>54.472767</td>\n",
+       "      <td>-1.654932</td>\n",
+       "      <td>True</td>\n",
+       "      <td>508670.609809</td>\n",
+       "      <td>422359.747233</td>\n",
+       "      <td>-1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>54.529720</td>\n",
+       "      <td>-1.667143</td>\n",
+       "      <td>True</td>\n",
+       "      <td>515003.452959</td>\n",
+       "      <td>421538.534748</td>\n",
+       "      <td>-1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>54.512981</td>\n",
+       "      <td>-1.589866</td>\n",
+       "      <td>True</td>\n",
+       "      <td>513167.311551</td>\n",
+       "      <td>426549.871569</td>\n",
+       "      <td>-1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>54.522320</td>\n",
+       "      <td>-1.380694</td>\n",
+       "      <td>True</td>\n",
+       "      <td>514305.528712</td>\n",
+       "      <td>440081.234190</td>\n",
+       "      <td>-1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>54.541656</td>\n",
+       "      <td>-1.613490</td>\n",
+       "      <td>True</td>\n",
+       "      <td>516349.193146</td>\n",
+       "      <td>425002.998690</td>\n",
+       "      <td>-1</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "         lat      long infected       northing        easting  cluster\n",
+       "0  54.472767 -1.654932     True  508670.609809  422359.747233       -1\n",
+       "1  54.529720 -1.667143     True  515003.452959  421538.534748       -1\n",
+       "2  54.512981 -1.589866     True  513167.311551  426549.871569       -1\n",
+       "3  54.522320 -1.380694     True  514305.528712  440081.234190       -1\n",
+       "4  54.541656 -1.613490     True  516349.193146  425002.998690       -1"
+      ]
+     },
+     "execution_count": 69,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "cupy_lat = cp.asarray(infected_df['lat'])\n",
+    "cupy_long = cp.asarray(infected_df['long'])\n",
+    "\n",
+    "infected_df['northing'], infected_df['easting'] = latlong2osgbgrid_cupy(cupy_lat, cupy_long)\n",
+    "infected_df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Find Clusters of Infected People"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Use DBSCAN to find clusters of at least 25 infected people where no member is more than 2000m from at least one other cluster member. Create a new column in `infected_df` which contains the cluster to which each infected person belongs."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 70,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<cudf.core.groupby.groupby.DataFrameGroupBy object at 0x7f55ea949240>"
+      ]
+     },
+     "execution_count": 70,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dbscan = cuml.DBSCAN(eps = 2000, min_samples = 25)\n",
+    "infected_df['cluster'] = dbscan.fit_predict(infected_df[['northing', 'easting']])\n",
+    "infected_df.groupby('cluster')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Find the Centroid of Each Cluster"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Use grouping to find the mean `northing` and `easting` values for each cluster identified above."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 71,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>northing</th>\n",
+       "      <th>easting</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>cluster</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>-1</th>\n",
+       "      <td>378094.622647</td>\n",
+       "      <td>401880.682473</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>397661.319575</td>\n",
+       "      <td>371410.021738</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>436475.527827</td>\n",
+       "      <td>332980.449214</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>347062.477357</td>\n",
+       "      <td>389386.823243</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>359668.552556</td>\n",
+       "      <td>379638.020362</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>391630.403390</td>\n",
+       "      <td>431158.137254</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>386471.397432</td>\n",
+       "      <td>426559.085587</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>434970.462486</td>\n",
+       "      <td>406985.278520</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>412772.652344</td>\n",
+       "      <td>410069.663793</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>415808.971615</td>\n",
+       "      <td>414713.750256</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>417322.530166</td>\n",
+       "      <td>409583.737652</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>334208.471668</td>\n",
+       "      <td>435937.777721</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>300568.023792</td>\n",
+       "      <td>391901.514790</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>291539.540205</td>\n",
+       "      <td>401640.663845</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>289855.069902</td>\n",
+       "      <td>394518.295606</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "              northing        easting\n",
+       "cluster                              \n",
+       "-1       378094.622647  401880.682473\n",
+       " 0       397661.319575  371410.021738\n",
+       " 1       436475.527827  332980.449214\n",
+       " 2       347062.477357  389386.823243\n",
+       " 3       359668.552556  379638.020362\n",
+       " 4       391630.403390  431158.137254\n",
+       " 5       386471.397432  426559.085587\n",
+       " 6       434970.462486  406985.278520\n",
+       " 7       412772.652344  410069.663793\n",
+       " 8       415808.971615  414713.750256\n",
+       " 9       417322.530166  409583.737652\n",
+       " 10      334208.471668  435937.777721\n",
+       " 11      300568.023792  391901.514790\n",
+       " 12      291539.540205  401640.663845\n",
+       " 13      289855.069902  394518.295606"
+      ]
+     },
+     "execution_count": 71,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "centroids_df = infected_df[['northing', 'easting', 'cluster']].groupby('cluster').mean()\n",
+    "centroids_df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Find the number of people in each cluster by counting the number of appearances of each cluster's label in the column produced by DBSCAN."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 72,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "cluster\n",
+       "-1     8451\n",
+       " 0     8638\n",
+       " 1       68\n",
+       " 2      403\n",
+       " 3       25\n",
+       " 4       66\n",
+       " 5       43\n",
+       " 6       27\n",
+       " 7       39\n",
+       " 8       92\n",
+       " 9       21\n",
+       " 10      64\n",
+       " 11      68\n",
+       " 12      72\n",
+       " 13      71\n",
+       "dtype: int64"
+      ]
+     },
+     "execution_count": 72,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "infected_df.groupby(['cluster']).size()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Find the Centroid of the Cluster with the Most Members ##"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Use the cluster label for with the most people to filter `centroid_df` and write the answer to `my_assessment/question_1.json`. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 78,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/conda/lib/python3.10/site-packages/cudf/io/json.py:194: UserWarning: Using CPU via Pandas to write JSON dataset\n",
+      "  warnings.warn(\"Using CPU via Pandas to write JSON dataset\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "centroids_df.loc[0].to_json('my_assessment/question_1.json')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Check Submission ##"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 79,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\"northing\":397661.3195752321,\"easting\":371410.0217381102}"
+     ]
+    }
+   ],
+   "source": [
+    "!cat my_assessment/question_1.json"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Tip**: Your submission file should contain one line of text, similar to: \n",
+    "\n",
+    "```\n",
+    "{'northing':XXX.XX,'easting':XXX.XX}\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<div align=\"center\"><h2>Please Restart the Kernel</h2></div>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import IPython\n",
+    "app = IPython.Application.instance()\n",
+    "app.kernel.do_shutdown(True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<a href=\"https://www.nvidia.com/dli\"><img src=\"images/DLI_Header.png\" alt=\"Header\" style=\"width: 400px;\"/></a>"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

5/data science/3/2_04_embedding.ipynb

@@ -0,0 +1,776 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "0bf7f930-76a1-4c16-84e4-cf1e73b54c55",
+   "metadata": {},
+   "source": [
+    "<a href=\"https://www.nvidia.com/dli\"> <img src=\"images/DLI_Header.png\" alt=\"Header\" style=\"width: 400px;\"/> </a>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "400a41da-bc38-4e9a-9ece-d2744ffb16b0",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "# Enhancing Data Science Outcomes With Efficient Workflow #"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8897c66c-4f9d-48b4-a60b-ddae16f2f61b",
+   "metadata": {},
+   "source": [
+    "## 04 - Embeddings ##\n",
+    "In this lab, you will use high-performance computing to create machine learning solutions. This lab covers the model development portion of the data science workflow. A good machine learning solution excels that both accuracy and inference performance. \n",
+    "\n",
+    "<p><img src='images/pipeline_overview_2.png' width=1080></p>\n",
+    "\n",
+    "**Table of Contents**\n",
+    "<br>\n",
+    "This notebook covers the below sections: \n",
+    "1. [Entity Embedding](#s4-1)\n",
+    "2. [Training the Embeddings](#s4-2)\n",
+    "    * [Preparing the Data - Normalization](#s4-2.1)\n",
+    "    * [Model Building](#s4-2.2)\n",
+    "    * [Being Training](#s4-2.3)\n",
+    "3. [Visualizing the Embeddings](#s4-3)\n",
+    "4. [Conclusion](#s4-4)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "28538773-6b95-4840-aca2-73a6f7d98b07",
+   "metadata": {},
+   "source": [
+    "<a name='s4-1'></a>\n",
+    "## Entity Embeddings ##\n",
+    "[Entity Embeddings](https://arxiv.org/pdf/1604.06737.pdf) are very similar to word embeddings used in NLP. They are a way to represent categorical features in a defined latent space. In the latent space, categories that are semantically similar have similar vectors. Embeddings can be trained to assign a learnable feature vector to each category. Using embeddings, each categorical value is mapped to its own associated vector representation that is more informative than a single point value. Even though embeddings require a large amount of data and computational resources to train, they have proven to be a great alternative encoding method to consider. Once trained, embeddings can boost the performance of downstream machine learning tasks when used as the input features. Users can combine the power of deep learning with traditional machine learning on tabular data. \n",
+    "\n",
+    "<p><img src='images/embedding.png' width=720></p>\n",
+    "\n",
+    "Reasons for using embeddings include: \n",
+    "* It is much more efficient than the one-hot approach for encoding when cardinality if high\n",
+    "* Allows rich relationships and complexities between categories to be captured\n",
+    "* Reduce memory usage and speed up downstream machine learning model training\n",
+    "* Once trained, the same embedding can be used for various use cases\n",
+    "* Can be used to visualize categorical data and for data clustering, since the embedding space quantifies semantic similarity as distance between the categories in the latent space\n",
+    "* Mitigates the need to perform cumbersome manual feature engineering, which requires extensive domain knowledge\n",
+    "\n",
+    "<p><img src='images/tip.png' width=720></p>\n",
+    "\n",
+    "Below are some tips about embeddings: \n",
+    "* Requires training with large amounts of data, making it inappropriate for unseen data such as when new categories are added\n",
+    "* Can overfit\n",
+    "* Difficult to interpret"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6ba4160d-4b41-40d3-93bc-f1fae0b9dddc",
+   "metadata": {},
+   "source": [
+    "<a name='s4-2'></a>\n",
+    "## Training the Embeddings ##\n",
+    "Embeddings aim to represent each entity as a numeric vector such that products in similar context have similar vectors. Mathematically, similar entities will have a large dot product whereas every entity when one-hot encoded has a zero dot product with every other entity. This is because all one-hot vectors are orthogonal. \n",
+    "\n",
+    "We will use [PyTorch](https://pytorch.org/) to train a simple fully-connected neural network. A surrogate problem is setup for the purpose of finding the embedding vectors. Neural networks have difficultly with sparse categorical features. Traditionally, embeddings are a way to reduce those features to increase model performance. \n",
+    "\n",
+    "Technically, the idea of an embedding layer is very similar to a dense or linear layer (without bias) in the neural network. When training an embedding this way, users will one-hot encode the categorical data so each record becomes a vector with C features, where C is the cardinality. We then perform matrix vector multiplication on the input vector and the weights before feeding the next layer. This is inefficient when the number of input features is large and sparse, as is the case for categorical features from a tabular dataset. \n",
+    "\n",
+    "A better and more efficient approach would be to train a `torch.nn.Embedding` layer, which can be treated as a \"lookup\" table with the label-encoded category id as the index. By using choosing this, we avoid one-hot encoding and the matrix vector multiplication. \n",
+    "\n",
+    "<p><img src='images/surrogate_problem.png' width=720></p>\n",
+    "\n",
+    "<p><img src='images/tip.png' width=720></p>\n",
+    "\n",
+    "Embeddings will naturally be affected by how the surrogate problem is defined. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "ec50a570-247f-4cfc-8dc5-2c2b501de703",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# import dependencies\n",
+    "from tqdm import tqdm\n",
+    "import cudf\n",
+    "import cuml\n",
+    "import dask_cudf\n",
+    "\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import torch.nn.functional as F\n",
+    "import torch.optim as torch_optim\n",
+    "from torch.utils.data import Dataset, DataLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "036bf6ee-d5cb-4f20-a591-681706a098ac",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# set device cuda to use GPU\n",
+    "device=torch.device('cuda')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "3726fc69-2a2b-42e2-be12-d235ce2322c1",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# define features and label\n",
+    "cols=['brand', 'cat_0', 'cat_1', 'cat_2', 'price', 'target']\n",
+    "cat_cols=['brand', 'cat_0', 'cat_1', 'cat_2']\n",
+    "label='target'\n",
+    "\n",
+    "feature_cols=[col for col in cols if col != label]\n",
+    "cont_cols=[col for col in feature_cols if col not in cat_cols] # ['price']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "ae87d23f-0c67-4758-8842-ca5770e740f9",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Total of 2461697 records.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# read data\n",
+    "parquet_dir='processed_parquet'\n",
+    "\n",
+    "ddf=dask_cudf.read_parquet(parquet_dir, columns=cols)\n",
+    "gdf=ddf.compute()\n",
+    "\n",
+    "print(f'Total of {len(gdf)} records.')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b9110c9d-5924-4cb2-8bf3-cabd398aad0e",
+   "metadata": {},
+   "source": [
+    "<p><img src='images/tip.png' width=720></p>\n",
+    "\n",
+    "Even though we intend to keep all the data in one GPU, we still recommend loading data with `Dask-cuDF`. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "f782bc7e-e6c4-4d87-a839-5a99227dca7c",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>brand</th>\n",
+       "      <th>cat_0</th>\n",
+       "      <th>cat_1</th>\n",
+       "      <th>cat_2</th>\n",
+       "      <th>price</th>\n",
+       "      <th>target</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1</td>\n",
+       "      <td>6</td>\n",
+       "      <td>5</td>\n",
+       "      <td>2</td>\n",
+       "      <td>100.229996</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>871.839966</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>872.090027</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2</td>\n",
+       "      <td>6</td>\n",
+       "      <td>5</td>\n",
+       "      <td>2</td>\n",
+       "      <td>306.690002</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>13</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "      <td>24</td>\n",
+       "      <td>334.349976</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   brand  cat_0  cat_1  cat_2       price  target\n",
+       "0      1      6      5      2  100.229996       1\n",
+       "1      2      1      1      1  871.839966       1\n",
+       "2      2      1      1      1  872.090027       1\n",
+       "3      2      6      5      2  306.690002       1\n",
+       "4     13      2      3     24  334.349976       1"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "gdf.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "3673f202-7aea-43a7-a569-4c210a614529",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'brand': (3303, 7), 'cat_0': (14, 3), 'cat_1': (61, 3), 'cat_2': (90, 3)}"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# the embedding vectors will start with 0 so we decrease the categorical values by 1 to match\n",
+    "gdf[cat_cols]=gdf[cat_cols]-1\n",
+    "\n",
+    "n_uniques=gdf.nunique()\n",
+    "\n",
+    "# use higher of 4th root of nunique and 3 for vector dimension\n",
+    "embedding_sizes={col: (n_uniques[col], max(3, int(n_uniques[col]**0.25))) for col in cat_cols}\n",
+    "embedding_sizes"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a327c1f9-0683-45f1-90a6-6d4d4daa093c",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "<p><img src='images/tip.png' width=720></p>\n",
+    "\n",
+    "The size of embeddings can become very large. For example, large embeddings are usually needed for users and items for large platforms. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c1c7fee-dad0-4009-a55c-513465db8a7c",
+   "metadata": {},
+   "source": [
+    "<a name='s4-2.1'></a>\n",
+    "### Preparing the Data - Normalization ###\n",
+    "**Normalization** is required to enable neural networks to leverage numerical features. Tree-based models do not require normalization as they define the split independent of the scale of a feature. Without normalization, neural networks are difficult to train. The reason is that different numerical features have different scales. When we combine the features in a hidden layer, the different scales make it more difficult to extract patterns from it. \n",
+    "\n",
+    "<p><img src='images/tip.png' width=720></p>\n",
+    "\n",
+    "We will also implement a `torch.nn.BatchNorm1d`[[doc]](https://pytorch.org/docs/stable/generated/torch.nn.BatchNorm1d.html) layer to mitigate the exploding gradient problem. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "fb1840b3-a7d8-4b91-98ef-bddf59afd5e6",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# normalize data\n",
+    "gdf['price']=cuml.preprocessing.StandardScaler().fit_transform(gdf[['price']])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d6991948-f79a-4b51-b3a9-2571b2be5262",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "<a name='s4-2.2'></a>\n",
+    "### Model Building ###\n",
+    "We construct a model with several layers. The embeddings will be the same dimension as num_unique x vector_size. The embeddings will be concatenated, along with the continous variable(s), before they are fed into the next layer. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "35a8055b-8b7b-4fb8-8d3a-9f36fc03b171",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# define neural network with embedding layers\n",
+    "class ProductPurchaseModel(nn.Module):\n",
+    "    def __init__(self, embedding_sizes, n_cont):\n",
+    "        super().__init__()\n",
+    "        # make an embedding for each categorical feature\n",
+    "        # The `nn.Embedding` layer can be thought of as a lookup table where the key is \n",
+    "        # the category index and the value is the corresponding embedding vector\n",
+    "        self.embeddings=nn.ModuleList([nn.Embedding(n_categories, size) for n_categories, size in embedding_sizes.values()])\n",
+    "        \n",
+    "        # n_emb is the length of all embeddings combined\n",
+    "        n_emb=sum(e.embedding_dim for e in self.embeddings)\n",
+    "        \n",
+    "        self.n_emb=n_emb\n",
+    "        self.n_cont=n_cont\n",
+    "        self.emb_drop = nn.Dropout(0.6)\n",
+    "        \n",
+    "        # apply dropout, batch norm and linear layers\n",
+    "        self.bn1=nn.BatchNorm1d(self.n_cont)\n",
+    "        self.lin1=nn.Linear(self.n_emb + self.n_cont, 200)\n",
+    "        self.drop1=nn.Dropout(0.3)\n",
+    "        self.bn2=nn.BatchNorm1d(200)\n",
+    "        self.drop2=nn.Dropout(0.3)\n",
+    "        self.lin2=nn.Linear(200, 70)\n",
+    "        self.bn3=nn.BatchNorm1d(70)\n",
+    "        self.lin3=nn.Linear(70, 2)\n",
+    "\n",
+    "    def forward(self, X_cat, X_cont):\n",
+    "        # map each categorical feature to the embedding vector on its corresponding embedding layer\n",
+    "        x_1=[embedding(X_cat[:, idx]) for idx, embedding in enumerate(self.embeddings)]\n",
+    "        \n",
+    "        # concatenate all categorical embedding vectors together\n",
+    "        x_1=torch.cat(x_1, 1)\n",
+    "        \n",
+    "        # apply random drop out, normalization, and activation\n",
+    "        x_1=self.emb_drop(x_1)\n",
+    "        x_2=self.bn1(X_cont)\n",
+    "        \n",
+    "        # concatenate categorical embeddings to input layer from continuous variable(s)\n",
+    "        x_1=torch.cat([x_1, x_2], 1)\n",
+    "        \n",
+    "        # apply random drop out, normalization, and activation\n",
+    "        x_1=F.relu(self.lin1(x_1))\n",
+    "        x_1=self.drop1(x_1)\n",
+    "        x_1=self.bn2(x_1)\n",
+    "        x_1=F.relu(self.lin2(x_1))\n",
+    "        x_1=self.drop2(x_1)\n",
+    "        x_1=self.bn3(x_1)\n",
+    "        x_1=self.lin3(x_1)\n",
+    "        return x_1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c52e50a2-99b6-4a8c-aa65-5f11a7806c6e",
+   "metadata": {},
+   "source": [
+    "<p><img src='images/tip.png' width=720></p>\n",
+    "\n",
+    "Tabular data uses shallow models with huge embedding tables and few feed-forward layers. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "5b7d18b1-d29e-43d4-8091-3aba41968ebf",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "ProductPurchaseModel(\n",
+       "  (embeddings): ModuleList(\n",
+       "    (0): Embedding(3303, 7)\n",
+       "    (1): Embedding(14, 3)\n",
+       "    (2): Embedding(61, 3)\n",
+       "    (3): Embedding(90, 3)\n",
+       "  )\n",
+       "  (emb_drop): Dropout(p=0.6, inplace=False)\n",
+       "  (bn1): BatchNorm1d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
+       "  (lin1): Linear(in_features=17, out_features=200, bias=True)\n",
+       "  (drop1): Dropout(p=0.3, inplace=False)\n",
+       "  (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
+       "  (drop2): Dropout(p=0.3, inplace=False)\n",
+       "  (lin2): Linear(in_features=200, out_features=70, bias=True)\n",
+       "  (bn3): BatchNorm1d(70, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
+       "  (lin3): Linear(in_features=70, out_features=2, bias=True)\n",
+       ")"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# instantiate model\n",
+    "model=ProductPurchaseModel(embedding_sizes, len(cont_cols))\n",
+    "model.to(device)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f35dab8e-f1cd-484b-999e-b9e0f7e79edd",
+   "metadata": {},
+   "source": [
+    "Next, we define a `torch.utils.data.Dataset` class to be use by `torch.utils.data.DataLoader`. The Dataset is makes it easier to track separate categorical and continuous variables. The DatalLoader wraps an iterable around the Dataset to enable easy access to the samples. More information about Dataset and DataLoader can be found in quick PyTorch [guide](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html). "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "98f74906-7b79-4fda-8626-df17023ee512",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# define dataset\n",
+    "class myDataset(Dataset):\n",
+    "    def __init__(self, X, y, cat_cols, cont_cols):\n",
+    "        self.X_cat=torch.as_tensor(X.loc[:, cat_cols].copy().values.astype('int32'), device=device)\n",
+    "        self.X_cont=torch.as_tensor(X.loc[:, cont_cols].copy().values.astype('float32'), device=device)\n",
+    "        self.y=torch.as_tensor(y.astype('int64'), device=device)\n",
+    "        \n",
+    "    def __len__(self):\n",
+    "        return len(self.y)\n",
+    "    \n",
+    "    def __getitem__(self, idx): \n",
+    "        return self.X_cat[idx], self.X_cont[idx], self.y[idx]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "a0973509-6a11-49d8-b346-ab9ec8cfaef5",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# instantiate dataset\n",
+    "X_train=gdf[feature_cols]\n",
+    "y_train=gdf['target'].values\n",
+    "\n",
+    "train_ds=myDataset(X_train, y_train, cat_cols, cont_cols)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5336cfd0-39ed-4285-9b66-e4f5d1b7d75e",
+   "metadata": {},
+   "source": [
+    "<a name='s4-2.3'></a>\n",
+    "### Begin Training ###\n",
+    "We will set some parameters for training. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "0604708e-1c2c-485b-a029-eadd17356a03",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# set optimizer\n",
+    "def get_optimizer(model, lr = 0.001, wd = 0.0):\n",
+    "    parameters=filter(lambda p: p.requires_grad, model.parameters())\n",
+    "    optim=torch_optim.Adam(parameters, lr=lr, weight_decay=wd)\n",
+    "    return optim"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "39e0ce25-f65c-4330-98cc-34ee4b30bae4",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# define training function\n",
+    "def train_model(model, optim, train_dl):\n",
+    "    # set the model to training, which is useful for BatchNorm and Dropout layers that behave differently during training and evaluation\n",
+    "    model.train()\n",
+    "    total=0\n",
+    "    sum_loss=0\n",
+    "    \n",
+    "    # iterate through batches\n",
+    "    for b, (X_cat, X_cont, y) in enumerate(train_dl):\n",
+    "        batch=y.shape[0]\n",
+    "        \n",
+    "        # forward pass\n",
+    "        output=model(X_cat, X_cont)\n",
+    "        \n",
+    "        # calculate loss\n",
+    "        loss=F.cross_entropy(output, y)\n",
+    "        \n",
+    "        # zero out the gradients so the parameters update correctly, otherwise gradients would be combined with old\n",
+    "        optim.zero_grad()\n",
+    "        loss.backward()\n",
+    "        optim.step()\n",
+    "        \n",
+    "        # calculate total loss per batch\n",
+    "        total+=batch\n",
+    "        sum_loss+=batch*(loss.item())\n",
+    "    return sum_loss/total"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a60dd511-3121-4eb0-beb7-3a03d56de202",
+   "metadata": {},
+   "source": [
+    "Instantiate a `torch.utils.data.DataLoader` and begin training. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "5a25e4e6-f0b5-4bbc-8a1d-0eee74c7faaf",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# define training loop\n",
+    "def train_loop(model, epochs, lr=0.01, wd=0.0):\n",
+    "    # instantiate optimizer\n",
+    "    optim=get_optimizer(model, lr = lr, wd = wd)\n",
+    "    \n",
+    "    # iterate through number of epochs\n",
+    "    for i in tqdm(range(epochs)): \n",
+    "        loss=train_model(model, optim, train_dl)\n",
+    "        print(\"training loss: \", round(loss, 3))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "68b43459-fb0a-4c13-9371-7c15327ff624",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 33%|███▎      | 1/3 [00:28<00:57, 28.79s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "training loss:  0.666\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 67%|██████▋   | 2/3 [00:57<00:28, 28.67s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "training loss:  0.665\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "# define batch size and begin training\n",
+    "batch_size=1000\n",
+    "train_dl=DataLoader(train_ds, batch_size=batch_size, shuffle=True)\n",
+    "\n",
+    "train_loop(model, epochs=3, lr=0.05, wd=0.00001)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7d6656b3-3642-4279-b787-0c034c45b739",
+   "metadata": {},
+   "source": [
+    "<a name='s4-3'></a>\n",
+    "## Visualizing the Embeddings ##"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "20973ee4-a723-4931-bf50-8efffe275026",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# visualize embeddings\n",
+    "\n",
+    "# import dependencies\n",
+    "import plotly.express as px\n",
+    "import pandas as pd\n",
+    "\n",
+    "# pick category to visualize\n",
+    "category='brand'\n",
+    "\n",
+    "category_label=pd.read_parquet(f'categories/unique.{category}.parquet')[category]\n",
+    "category_label=category_label[1:]\n",
+    "\n",
+    "embeddings_idx=list(embedding_sizes.keys()).index(category)\n",
+    "embeddings=model.embeddings[embeddings_idx].weight.detach().cpu().numpy()\n",
+    "\n",
+    "fig=px.scatter_3d(\n",
+    "    x=embeddings[:, 0], \n",
+    "    y=embeddings[:, 1], \n",
+    "    z=embeddings[:, 2], \n",
+    "    text=category_label, \n",
+    "    height=720\n",
+    ")\n",
+    "fig.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "130a2b16-89e5-4eda-8155-014a75a3638e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# persist embeddings\n",
+    "!mkdir trained_embedding_weights\n",
+    "\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "\n",
+    "for idx, each_col in enumerate(cat_cols): \n",
+    "    weights=model.embeddings[idx].weight.detach().cpu().numpy()\n",
+    "    pd.DataFrame(weights).to_csv(f'trained_embedding_weights/{each_col}.csv', index=False)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bc7cce0e-6dcb-4d5a-82dd-e8074abaaaec",
+   "metadata": {},
+   "source": [
+    "<a name='s4-4'></a>\n",
+    "## Conclusion ##\n",
+    "Deep Learning is very good at feature extraction, which can be used for finding categorical embeddings. This is the advantage of using a Deep Learning approach, as it requires way less feature engineering and less dependent on domain knowledge. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "997bd6f7-9efb-4fee-b3d4-9d4454694c7b",
+   "metadata": {},
+   "source": [
+    "<a href=\"https://www.nvidia.com/dli\"> <img src=\"images/DLI_Header.png\" alt=\"Header\" style=\"width: 400px;\"/> </a>"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}