/*
 * Copyright (c) 2022-2023 Hewlett Packard Enterprise Development LP
 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only
 */
#include "config.h"
#include "cxip.h"

#define CXIP_WARN(...) _CXIP_WARN(FI_LOG_EP_CTRL, __VA_ARGS__)

static void cxip_rdzv_pte_cb(struct cxip_pte *pte, const union c_event *event)
{
	switch (pte->state) {
	case C_PTLTE_ENABLED:
		break;
	default:
		CXIP_FATAL("Unexpected state received: %u\n", pte->state);
	}
}

static bool cxip_rdzv_pte_append_done(struct cxip_rdzv_pte *pte,
				      int expected_success_count)
{
	if (ofi_atomic_get32(&pte->le_linked_success_count) ==
	    expected_success_count)
		return true;

	if (ofi_atomic_get32(&pte->le_linked_failure_count) != 0)
		return true;

	return false;
}

static int cxip_rdzv_pte_wait_append(struct cxip_rdzv_pte *pte,
				     uint32_t expected_count)
{
	int ret = FI_SUCCESS;

	/* Poll until the LE is linked or a failure occurs. */
	do {
		cxip_evtq_progress(&pte->txc->base.tx_evtq, true);
		sched_yield();
	} while (!cxip_rdzv_pte_append_done(pte, expected_count));

	if (ofi_atomic_get32(&pte->le_linked_failure_count)) {
		ret = -FI_EIO;
		CXIP_WARN("Failed to append zero byte put LE: %d:%s\n", ret,
			  fi_strerror(-ret));
	}

	return ret;
}

static void cxip_rdzv_pte_src_reqs_free(struct cxip_rdzv_match_pte *pte)
{
	int i;

	/* The corresponding LE is not freed using an unlink command. Instead,
	 * this logic relies on the freeing of the hardware PtlTE to release the
	 * LEs.
	 */
	for (i = 0; i < RDZV_SRC_LES; i++) {
		if (pte->src_reqs[i])
			cxip_evtq_req_free(pte->src_reqs[i]);
	}
}

/* caller should hold ep_obj->lock */
int cxip_rdzv_pte_src_req_alloc(struct cxip_rdzv_match_pte *pte, int lac)
{
	int ret;
	union cxip_match_bits mb;
	union cxip_match_bits ib;
	uint32_t le_flags;
	int expected_success_count;
	struct cxip_req *req;
	struct cxip_rdzv_pte *base = &pte->base_pte;

	/* Reuse a previously allocated request whenever possible. */
	if (pte->src_reqs[lac])
		return FI_SUCCESS;

	mb.raw = 0;
	mb.rdzv_lac = lac;
	ib.raw = ~0;
	ib.rdzv_lac = 0;
	le_flags = C_LE_UNRESTRICTED_BODY_RO | C_LE_UNRESTRICTED_END_RO |
		C_LE_OP_GET | C_LE_EVENT_UNLINK_DISABLE;

	req = cxip_evtq_req_alloc(&base->txc->base.tx_evtq, 1, base);
	if (!req) {
		ret = -FI_EAGAIN;
		CXIP_WARN("Failed to allocate %d rendezvous source request: %d:%s\n",
			  lac, ret, fi_strerror(-ret));
		return ret;
	}
	req->cb = cxip_rdzv_pte_src_cb;

	expected_success_count =
		ofi_atomic_get32(&base->le_linked_success_count) + 1;

	ret = cxip_pte_append(base->pte, 0, -1ULL, lac, C_PTL_LIST_PRIORITY,
			      req->req_id, mb.raw, ib.raw,
			      CXI_MATCH_ID_ANY, 0, le_flags, NULL,
			      base->txc->rx_cmdq, true);
	if (ret) {
		CXIP_WARN("Failed to issue %d rendezvous source request LE append: %d:%s\n",
			  lac, ret, fi_strerror(-ret));
		goto err_free_req;
	}

	ret = cxip_rdzv_pte_wait_append(base, expected_success_count);
	if (ret != FI_SUCCESS)
		goto err_free_req;

	pte->src_reqs[lac] = req;

	return FI_SUCCESS;

err_free_req:
	cxip_evtq_req_free(req);

	return ret;
}

static void cxip_rdzv_pte_zbp_req_free(struct cxip_rdzv_match_pte *pte)
{
	/* The corresponding LE is not freed using an unlink command. Instead,
	 * this logic relies on the freeing of the hardware PtlTE to release the
	 * LEs.
	 */
	cxip_evtq_req_free(pte->zbp_req);
}

static int cxip_rdzv_pte_zbp_req_alloc(struct cxip_rdzv_match_pte *pte)
{
	uint32_t le_flags = C_LE_UNRESTRICTED_BODY_RO |
		C_LE_UNRESTRICTED_END_RO | C_LE_OP_PUT |
		C_LE_EVENT_UNLINK_DISABLE;
	union cxip_match_bits mb = {
		.le_type = CXIP_LE_TYPE_ZBP,
	};
	union cxip_match_bits ib = {
		.tag = ~0,
		.tx_id = ~0,
		.cq_data = 1,
		.tagged = 1,
		.match_comp = 1,
		.rdzv_done = 1,
	};
	struct cxip_rdzv_pte *base = &pte->base_pte;
	int ret;
	int expected_success_count;

	pte->zbp_req = cxip_evtq_req_alloc(&base->txc->base.tx_evtq, 1, pte);
	if (!pte->zbp_req) {
		ret = -FI_ENOMEM;
		CXIP_WARN("Failed to allocate zero byte put request: %d:%s\n",
			  ret, fi_strerror(-ret));
		return ret;
	}

	pte->zbp_req->cb = cxip_rdzv_pte_zbp_cb;

	expected_success_count =
		ofi_atomic_get32(&base->le_linked_success_count) + 1;

	ret = cxip_pte_append(base->pte, 0, 0, 0, C_PTL_LIST_PRIORITY,
			      pte->zbp_req->req_id, mb.raw, ib.raw,
			      CXI_MATCH_ID_ANY, 0, le_flags, NULL,
			      base->txc->rx_cmdq, true);
	if (ret) {
		CXIP_WARN("Failed to issue zero byte put LE append: %d:%s\n",
			  ret, fi_strerror(-ret));
		goto err_free_req;
	}

	ret = cxip_rdzv_pte_wait_append(base, expected_success_count);
	if (ret != FI_SUCCESS)
		goto err_free_req;

	return FI_SUCCESS;

err_free_req:
	cxip_evtq_req_free(pte->zbp_req);

	return ret;
}

static void cxip_rdzv_pte_free(struct cxip_rdzv_pte *pte)
{
	/* Freeing the PtlTE causes the PtlTE to be reset and all LEs to be
	 * freed. Thus, no need to issue disable and/or unlink commands.
	 */
	cxip_pte_free(pte->pte);

	/* Flush the CQ to ensure any events referencing the rendezvous requests
	 * are processed.
	 */
	cxip_evtq_progress(&pte->txc->base.tx_evtq, true);
}

void cxip_rdzv_match_pte_free(struct cxip_rdzv_match_pte *pte)
{
	cxip_rdzv_pte_free(&pte->base_pte);

	/* Release all the rendezvous requests. */
	cxip_rdzv_pte_src_reqs_free(pte);
	cxip_rdzv_pte_zbp_req_free(pte);

	free(pte);
}

void cxip_rdzv_nomatch_pte_free(struct cxip_rdzv_nomatch_pte *pte)
{
	cxip_rdzv_pte_free(&pte->base_pte);
	cxip_evtq_req_free(pte->le_req);

	free(pte);
}

static int cxip_rdzv_base_pte_alloc(struct cxip_txc_hpc *txc,
				    uint32_t write_pid_idx, bool write,
				    uint32_t read_pid_idx, bool read,
				    bool matching,
				    struct cxip_rdzv_pte *base_pte)
{
	int ret;
	struct cxi_pt_alloc_opts pt_opts = {
		.is_matching = matching,
	};

	base_pte->txc = txc;
	ofi_atomic_initialize32(&base_pte->le_linked_success_count, 0);
	ofi_atomic_initialize32(&base_pte->le_linked_failure_count, 0);

	if (matching && txc->base.ep_obj->av->symmetric)
		pt_opts.use_logical = 1;

	/* Reserve the Rendezvous Send PTE */
	ret = cxip_pte_alloc_nomap(txc->base.ep_obj->ptable,
				   txc->base.tx_evtq.eq,
				   &pt_opts, cxip_rdzv_pte_cb, txc,
				   &base_pte->pte);
	if (ret != FI_SUCCESS) {
		CXIP_WARN("Failed to alloc base rendezvous PtlTE: %d:%s\n",
			  ret, fi_strerror(-ret));
		return ret;
	}

	if (write) {
		ret = cxip_pte_map(base_pte->pte, write_pid_idx, false);
		if (ret != FI_SUCCESS) {
			CXIP_WARN("Failed to map write PTE: %d\n", ret);
			goto err_free_rdzv_pte;
		}
	}

	if (read) {
		ret = cxip_pte_map(base_pte->pte, read_pid_idx, false);
		if (ret != FI_SUCCESS) {
			CXIP_WARN("Failed to map read PTE: %d\n", ret);
			goto err_free_rdzv_pte;
		}
	}

	/* Set to enable, event will be processed on link */
	ret = cxip_pte_set_state(base_pte->pte, txc->rx_cmdq,
				 C_PTLTE_ENABLED, CXIP_PTE_IGNORE_DROPS);
	if (ret != FI_SUCCESS) {
		CXIP_WARN("Failed to enqueue enable command: %d:%s\n", ret,
			  fi_strerror(-ret));
		goto err_free_rdzv_pte;
	}

	return FI_SUCCESS;

err_free_rdzv_pte:
	cxip_pte_free(base_pte->pte);
	base_pte->pte = NULL;

	return ret;
}

/* ep_obj->lock should be held by caller */
int cxip_rdzv_match_pte_alloc(struct cxip_txc_hpc *txc,
			      struct cxip_rdzv_match_pte **rdzv_pte)
{
	int ret;
	struct cxip_rdzv_match_pte *match_pte;
	uint32_t pid_idx = txc->base.domain->iface->dev->info.rdzv_get_idx;
	struct cxip_rdzv_pte *base;

	match_pte = calloc(1, sizeof(*match_pte));
	if (!match_pte) {
		ret = -ENOMEM;
		CXIP_WARN("Rendzvous matching PtlTE allocation error: %d:%s\n",
			  ret, fi_strerror(-ret));
		return ret;
	}

	ret = cxip_rdzv_base_pte_alloc(txc, pid_idx, true,
				       CXIP_PTL_IDX_RDZV_DEST, true,
				       true, &match_pte->base_pte);
	if (ret != FI_SUCCESS)
		goto err_free_rdzv_pte_mem;

	/* Matching specific initialization */
	base = &match_pte->base_pte;

	ret = cxip_rdzv_pte_zbp_req_alloc(match_pte);
	if (ret) {
		CXIP_WARN("Failed to allocate zero byte put request: %d:%s\n",
			  ret, fi_strerror(-ret));
		goto err_free_rdzv_pte;
	}
	*rdzv_pte = match_pte;

	return FI_SUCCESS;

err_free_rdzv_pte:
	cxip_pte_free(base->pte);
err_free_rdzv_pte_mem:
	free(match_pte);

	return ret;
}

/* ep_obj->lock should be held by caller */
int cxip_rdzv_nomatch_pte_alloc(struct cxip_txc_hpc *txc, int lac,
				struct cxip_rdzv_nomatch_pte **rdzv_pte)
{
	int ret;
	struct cxip_rdzv_nomatch_pte *nomatch_pte;
	struct cxip_rdzv_pte *base;
	uint32_t le_flags;
	uint32_t expected_success_count;
	uint64_t ib = 0;
	uint32_t pid_idx = CXIP_PTL_IDX_RDZV_RESTRICTED(lac);

#if ENABLE_DEBUG
	/* Enable testing of fallback to default rendezvous protocol
	 * if unable to allocate required non-matching PTE/LE resources.
	 */
	if (txc->force_err & CXIP_TXC_FORCE_ERR_ALT_READ_PROTO_ALLOC)
		return -FI_ENOSPC;
#endif
	nomatch_pte = calloc(1, sizeof(*nomatch_pte));
	if (!nomatch_pte) {
		ret = -ENOMEM;
		CXIP_WARN("Rdzv restricted PtlTE allocation error: %d:%s\n",
			  ret, fi_strerror(-ret));
		return ret;
	}

	ret = cxip_rdzv_base_pte_alloc(txc, 0, false, pid_idx, true,
				       false, &nomatch_pte->base_pte);
	if (ret != FI_SUCCESS)
		goto err_free_rdzv_pte_mem;

	/* Non-matching specific initialization */
	base = &nomatch_pte->base_pte;

	nomatch_pte->le_req = cxip_evtq_req_alloc(&txc->base.tx_evtq, 1,
						  nomatch_pte);
	if (!nomatch_pte->le_req) {
		ret = -FI_EAGAIN;
		CXIP_WARN("Rdzv PtlTE LAC %d request allocation error: %d:%s\n",
			  lac, ret, fi_strerror(-ret));
		goto err_free_rdzv_pte;
	}

	nomatch_pte->le_req->cb = cxip_rdzv_pte_src_cb;

	le_flags = C_LE_EVENT_COMM_DISABLE | C_LE_EVENT_SUCCESS_DISABLE |
		   C_LE_UNRESTRICTED_BODY_RO | C_LE_OP_GET;
	ib = 1;
	expected_success_count =
		ofi_atomic_get32(&base->le_linked_success_count) + 1;

	ret = cxip_pte_append(base->pte, 0, -1ULL, lac, C_PTL_LIST_PRIORITY,
			      nomatch_pte->le_req->req_id, 0, ib,
			      CXI_MATCH_ID_ANY, 0, le_flags, NULL,
			      txc->rx_cmdq, true);
	if (ret != FI_SUCCESS) {
		CXIP_WARN("Failed to enqueue append cmd: %d:%s\n",
			  ret, fi_strerror(-ret));
		goto err_free_rdzv_pte;
	}

	ret = cxip_rdzv_pte_wait_append(base, expected_success_count);
	if (ret != FI_SUCCESS)
		goto err_free_req;

	*rdzv_pte = nomatch_pte;
	return FI_SUCCESS;

err_free_req:
	cxip_evtq_req_free(nomatch_pte->le_req);
err_free_rdzv_pte:
	cxip_pte_free(nomatch_pte->base_pte.pte);
err_free_rdzv_pte_mem:
	free(nomatch_pte);

	return ret;
}
