MPI 矩阵-矩阵乘法的问题：集群比单台计算机慢

Question

我使用 MPI 编写了一个小程序来并行化矩阵-矩阵乘法。问题是：在我的电脑上运行程序时，大约需要10秒才能完成，但在集群上大约需要75秒。我想我有一些同步问题，但我无法弄清楚（还）。

这是我的源代码：

/*matrix.c
mpicc -o out matrix.c
mpirun -np 11 out
*/

#include <mpi.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#define N 1000

#define DATA_TAG 10
#define B_SENT_TAG 20
#define FINISH_TAG 30

int master(int);
int worker(int, int);

int main(int argc, char **argv) {
    int myrank, p;
    double s_time, f_time;

    MPI_Init(&argc,&argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
    MPI_Comm_size(MPI_COMM_WORLD, &p);

    if (myrank == 0) {
        s_time = MPI_Wtime();
        master(p);
        f_time = MPI_Wtime();
        printf("Complete in %1.2f seconds\n", f_time - s_time);
        fflush(stdout);
    }
    else {
        worker(myrank, p);
    }
    MPI_Finalize();
    return 0;
}

int *read_matrix_row();
int *read_matrix_col();
int send_row(int *, int);
int recv_row(int *, int, MPI_Status *);
int send_tag(int, int);
int write_matrix(int *);

int master(int p) {
    MPI_Status status;
    int *a; int *b;
    int *c = (int *)malloc(N * sizeof(int));
    int i, j; int num_of_finish_row = 0;

    while (1) {
        for (i = 1; i < p; i++) {
            a = read_matrix_row();
            b = read_matrix_col();
            send_row(a, i);
            send_row(b, i);
            //printf("Master - Send data to worker %d\n", i);fflush(stdout);
        }
        wait();
        for (i = 1; i < N / (p - 1); i++) {
            for (j = 1; j < p; j++) {
                //printf("Master - Send next row to worker[%d]\n", j);fflush(stdout);
                b = read_matrix_col();
                send_row(b, j);
            }
        }
        for (i = 1; i < p; i++) {
            //printf("Master - Announce all row of B sent to worker[%d]\n", i);fflush(stdout);
            send_tag(i, B_SENT_TAG);
        }
        //MPI_Barrier(MPI_COMM_WORLD);
        for (i = 1; i < p; i++) {
            recv_row(c, MPI_ANY_SOURCE, &status);
            //printf("Master - Receive result\n");fflush(stdout);
            num_of_finish_row++;
        }
        //printf("Master - Finish %d rows\n", num_of_finish_row);fflush(stdout);
        if (num_of_finish_row >= N)
            break;
    }
    //printf("Master - Finish multiply two matrix\n");fflush(stdout);
    for (i = 1; i < p; i++) {
        send_tag(i, FINISH_TAG);
    }
    //write_matrix(c);
    return 0;
}

int worker(int myrank, int p) {
    int *a = (int *)malloc(N * sizeof(int));
    int *b = (int *)malloc(N * sizeof(int));
    int *c = (int *)malloc(N * sizeof(int));
    int i;
    for (i = 0; i < N; i++) {
        c[i] = 0;
    }
    MPI_Status status;
    int next = (myrank == (p - 1)) ? 1 : myrank + 1;
    int prev = (myrank == 1) ? p - 1 : myrank - 1;
    while (1) {
        recv_row(a, 0, &status);
        if (status.MPI_TAG == FINISH_TAG)
            break;
        recv_row(b, 0, &status);
        wait();
        //printf("Worker[%d] - Receive data from master\n", myrank);fflush(stdout);
        while (1) {
            for (i = 1; i < p; i++) {
                //printf("Worker[%d] - Start calculation\n", myrank);fflush(stdout);
                calc(c, a, b);
                //printf("Worker[%d] - Exchange data with %d, %d\n", myrank, next, prev);fflush(stdout);
                exchange(b, next, prev);
            }
            //printf("Worker %d- Request for more B's row\n", myrank);fflush(stdout);
            recv_row(b, 0, &status);
            //printf("Worker %d - Receive tag %d\n", myrank, status.MPI_TAG);fflush(stdout);
            if (status.MPI_TAG == B_SENT_TAG) {
                break;
                //printf("Worker[%d] - Finish calc one row\n", myrank);fflush(stdout);
            }
        }
        //wait();
        //printf("Worker %d - Send result\n", myrank);fflush(stdout);
        send_row(c, 0);
        for (i = 0; i < N; i++) {
            c[i] = 0;
        }
    }
    return 0;
}

int *read_matrix_row() {
    int *row = (int *)malloc(N * sizeof(int));
    int i;
    for (i = 0; i < N; i++) {
        row[i] = 1;
    }
    return row;
}
int *read_matrix_col() {
    int *col = (int *)malloc(N * sizeof(int));
    int i;
    for (i = 0; i < N; i++) {
        col[i] = 1;
    }
    return col;
}

int send_row(int *row, int dest) {
    MPI_Send(row, N, MPI_INT, dest, DATA_TAG, MPI_COMM_WORLD);
    return 0;
}

int recv_row(int *row, int src, MPI_Status *status) {
    MPI_Recv(row, N, MPI_INT, src, MPI_ANY_TAG, MPI_COMM_WORLD, status);
    return 0;
}

int wait() {
    MPI_Barrier(MPI_COMM_WORLD);
    return 0;
}
int calc(int *c_row, int *a_row, int *b_row) {
    int i;
    for (i = 0; i < N; i++) {
        c_row[i] = c_row[i] + a_row[i] * b_row[i];
        //printf("%d ", c_row[i]);
    }
    //printf("\n");fflush(stdout);
    return 0;
}

int exchange(int *row, int next, int prev) {
    MPI_Request request; MPI_Status status;
    MPI_Isend(row, N, MPI_INT, next, DATA_TAG, MPI_COMM_WORLD, &request);
    MPI_Irecv(row, N, MPI_INT, prev, MPI_ANY_TAG, MPI_COMM_WORLD, &request);
    MPI_Wait(&request, &status);
    return 0;
}

int send_tag(int worker, int tag) {
    MPI_Send(0, 0, MPI_INT, worker, tag, MPI_COMM_WORLD);
    return 0;
}

int write_matrix(int *matrix) {
    int i;
    for (i = 0; i < N; i++) {
        printf("%d ", matrix[i]);
    }
    printf("\n");
    fflush(stdout);
    return 0;
}

Answer 1

好吧，你有一个相当小的矩阵（N = 1000），其次你在行/列的基础上分布你的算法而不是阻塞。

对于使用更好算法的更现实的版本，您可能需要获取优化的 BLAS 库（例如 GOTO 是免费的），用该库测试单线程性能，然后获取 PBLAS 并将其与优化的 BLAS 链接，并比较 MPI 并行性能使用 PBLAS 版本。