remove odo catalog command (#5577)

* remove odo catalog command

- move devfile registry relevant code to pkg/registry

* remove odo catalog page form docs

* remove a few forgotten uses of catalog from clientset.go

* fix typos in docs

pkg/registry/devfile_component_type_list.go

@@ -0,0 +1,35 @@
+package registry
+
+import "sort"
+
+// GetLanguages returns the list of unique languages, ordered by name,
+// from a list of registry items
+func (o *DevfileStackList) GetLanguages() []string {
+	languagesMap := map[string]bool{}
+	for _, item := range o.Items {
+		languagesMap[item.Language] = true
+	}
+
+	languages := make([]string, 0, len(languagesMap))
+	for k := range languagesMap {
+		languages = append(languages, k)
+	}
+	sort.Strings(languages)
+	return languages
+}
+
+// GetProjectTypes returns the list of project types and associated details
+// from a list of registry items
+func (o *DevfileStackList) GetProjectTypes(language string) TypesWithDetails {
+	types := TypesWithDetails{}
+	for _, item := range o.Items {
+		if item.Language != language {
+			continue
+		}
+		if _, found := types[item.DisplayName]; !found {
+			types[item.DisplayName] = []DevfileStack{}
+		}
+		types[item.DisplayName] = append(types[item.DisplayName], item)
+	}
+	return types
+}

pkg/registry/devfile_component_type_list_test.go

@@ -0,0 +1,182 @@
+package registry
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestDevfileStackList_GetLanguages(t *testing.T) {
+	type fields struct {
+		Items []DevfileStack
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   []string
+	}{
+		{
+			name: "no devfiles",
+			want: []string{},
+		},
+		{
+			name: "some devfiles",
+			fields: fields{
+				Items: []DevfileStack{
+					{
+						Name:        "devfile4",
+						DisplayName: "first devfile for lang3",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+						Language: "lang3",
+					},
+					{
+						Name:        "devfile1",
+						DisplayName: "first devfile for lang1",
+						Registry: Registry{
+							Name: "Registry2",
+						},
+						Language: "lang1",
+					},
+					{
+						Name:        "devfile3",
+						DisplayName: "another devfile for lang2",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+						Language: "lang2",
+					},
+					{
+						Name:        "devfile2",
+						DisplayName: "second devfile for lang1",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+						Language: "lang1",
+					},
+				},
+			},
+			want: []string{"lang1", "lang2", "lang3"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			o := &DevfileStackList{
+				Items: tt.fields.Items,
+			}
+			if got := o.GetLanguages(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("DevfileStackList.GetLanguages() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestDevfileStackList_GetProjectTypes(t *testing.T) {
+	type fields struct {
+		Items []DevfileStack
+	}
+	type args struct {
+		language string
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   TypesWithDetails
+	}{
+		{
+			name: "No devfiles => no project types",
+			want: TypesWithDetails{},
+		},
+		{
+			name: "project types for lang1",
+			fields: fields{
+				Items: []DevfileStack{
+					{
+						Name:        "devfile4",
+						DisplayName: "first devfile for lang3",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+						Language: "lang3",
+					},
+					{
+						Name:        "devfile1",
+						DisplayName: "first devfile for lang1",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+						Language: "lang1",
+					},
+					{
+						Name:        "devfile1",
+						DisplayName: "first devfile for lang1",
+						Registry: Registry{
+							Name: "Registry2",
+						},
+						Language: "lang1",
+					},
+					{
+						Name:        "devfile3",
+						DisplayName: "another devfile for lang2",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+						Language: "lang2",
+					},
+					{
+						Name:        "devfile2",
+						DisplayName: "second devfile for lang1",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+						Language: "lang1",
+					},
+				},
+			},
+			args: args{
+				language: "lang1",
+			},
+			want: TypesWithDetails{
+				"first devfile for lang1": []DevfileStack{
+					{
+						Name:        "devfile1",
+						DisplayName: "first devfile for lang1",
+						Language:    "lang1",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+					{
+						Name:        "devfile1",
+						DisplayName: "first devfile for lang1",
+						Language:    "lang1",
+						Registry: Registry{
+							Name: "Registry2",
+						},
+					},
+				},
+				"second devfile for lang1": []DevfileStack{
+					{
+						Name:        "devfile2",
+						DisplayName: "second devfile for lang1",
+						Language:    "lang1",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			o := &DevfileStackList{
+				Items: tt.fields.Items,
+			}
+			if got := o.GetProjectTypes(tt.args.language); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("DevfileStackList.GetProjectTypes() = \n%+v, want \n%+v", got, tt.want)
+			}
+		})
+	}
+}

pkg/registry/interface.go

@@ -0,0 +1,16 @@
+// package registry wraps various package level functions into a Client interface to be able to mock them
+package registry
+
+import (
+	devfilev1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
+	dfutil "github.com/devfile/library/pkg/util"
+	"github.com/devfile/registry-support/registry-library/library"
+)
+
+type Client interface {
+	PullStackFromRegistry(registry string, stack string, destDir string, options library.RegistryOptions) error
+	DownloadFileInMemory(params dfutil.HTTPRequestParams) ([]byte, error)
+	DownloadStarterProject(starterProject *devfilev1.StarterProject, decryptedToken string, contextDir string, verbose bool) error
+	GetDevfileRegistries(registryName string) ([]Registry, error)
+	ListDevfileStacks(registryName string) (DevfileStackList, error)
+}

pkg/registry/mock.go

@@ -0,0 +1,110 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: pkg/registry/interface.go
+
+// Package registry is a generated GoMock package.
+package registry
+
+import (
+	reflect "reflect"
+
+	v1alpha2 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
+	util "github.com/devfile/library/pkg/util"
+	library "github.com/devfile/registry-support/registry-library/library"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockClient is a mock of Client interface.
+type MockClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockClientMockRecorder
+}
+
+// MockClientMockRecorder is the mock recorder for MockClient.
+type MockClientMockRecorder struct {
+	mock *MockClient
+}
+
+// NewMockClient creates a new mock instance.
+func NewMockClient(ctrl *gomock.Controller) *MockClient {
+	mock := &MockClient{ctrl: ctrl}
+	mock.recorder = &MockClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockClient) EXPECT() *MockClientMockRecorder {
+	return m.recorder
+}
+
+// DownloadFileInMemory mocks base method.
+func (m *MockClient) DownloadFileInMemory(params util.HTTPRequestParams) ([]byte, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DownloadFileInMemory", params)
+	ret0, _ := ret[0].([]byte)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// DownloadFileInMemory indicates an expected call of DownloadFileInMemory.
+func (mr *MockClientMockRecorder) DownloadFileInMemory(params interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DownloadFileInMemory", reflect.TypeOf((*MockClient)(nil).DownloadFileInMemory), params)
+}
+
+// DownloadStarterProject mocks base method.
+func (m *MockClient) DownloadStarterProject(starterProject *v1alpha2.StarterProject, decryptedToken, contextDir string, verbose bool) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DownloadStarterProject", starterProject, decryptedToken, contextDir, verbose)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DownloadStarterProject indicates an expected call of DownloadStarterProject.
+func (mr *MockClientMockRecorder) DownloadStarterProject(starterProject, decryptedToken, contextDir, verbose interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DownloadStarterProject", reflect.TypeOf((*MockClient)(nil).DownloadStarterProject), starterProject, decryptedToken, contextDir, verbose)
+}
+
+// GetDevfileRegistries mocks base method.
+func (m *MockClient) GetDevfileRegistries(registryName string) ([]Registry, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDevfileRegistries", registryName)
+	ret0, _ := ret[0].([]Registry)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetDevfileRegistries indicates an expected call of GetDevfileRegistries.
+func (mr *MockClientMockRecorder) GetDevfileRegistries(registryName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDevfileRegistries", reflect.TypeOf((*MockClient)(nil).GetDevfileRegistries), registryName)
+}
+
+// ListDevfileStacks mocks base method.
+func (m *MockClient) ListDevfileStacks(registryName string) (DevfileStackList, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListDevfileStacks", registryName)
+	ret0, _ := ret[0].(DevfileStackList)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListDevfileStacks indicates an expected call of ListDevfileStacks.
+func (mr *MockClientMockRecorder) ListDevfileStacks(registryName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDevfileStacks", reflect.TypeOf((*MockClient)(nil).ListDevfileStacks), registryName)
+}
+
+// PullStackFromRegistry mocks base method.
+func (m *MockClient) PullStackFromRegistry(registry, stack, destDir string, options library.RegistryOptions) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "PullStackFromRegistry", registry, stack, destDir, options)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// PullStackFromRegistry indicates an expected call of PullStackFromRegistry.
+func (mr *MockClientMockRecorder) PullStackFromRegistry(registry, stack, destDir, options interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PullStackFromRegistry", reflect.TypeOf((*MockClient)(nil).PullStackFromRegistry), registry, stack, destDir, options)
+}

pkg/registry/registry.go

@@ -0,0 +1,245 @@
+package registry
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strings"
+	"sync"
+
+	devfilev1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
+	dfutil "github.com/devfile/library/pkg/util"
+	indexSchema "github.com/devfile/registry-support/index/generator/schema"
+	"github.com/devfile/registry-support/registry-library/library"
+	registryUtil "github.com/redhat-developer/odo/pkg/odo/cli/preference/registry/util"
+	"github.com/zalando/go-keyring"
+
+	"github.com/redhat-developer/odo/pkg/component"
+	"github.com/redhat-developer/odo/pkg/log"
+	"github.com/redhat-developer/odo/pkg/preference"
+	"github.com/redhat-developer/odo/pkg/segment"
+	"github.com/redhat-developer/odo/pkg/testingutil/filesystem"
+	"github.com/redhat-developer/odo/pkg/util"
+)
+
+type RegistryClient struct {
+	fsys             filesystem.Filesystem
+	preferenceClient preference.Client
+}
+
+func NewRegistryClient(fsys filesystem.Filesystem, preferenceClient preference.Client) RegistryClient {
+	return RegistryClient{
+		fsys:             fsys,
+		preferenceClient: preferenceClient,
+	}
+}
+
+// PullStackFromRegistry pulls stack from registry with all stack resources (all media types) to the destination directory
+func (o RegistryClient) PullStackFromRegistry(registry string, stack string, destDir string, options library.RegistryOptions) error {
+	return library.PullStackFromRegistry(registry, stack, destDir, options)
+}
+
+// DownloadFileInMemory uses the url to download the file and return bytes
+func (o RegistryClient) DownloadFileInMemory(params dfutil.HTTPRequestParams) ([]byte, error) {
+	return util.DownloadFileInMemory(params)
+}
+
+// DownloadStarterProject downloads a starter project referenced in devfile
+// This will first remove the content of the contextDir
+func (o RegistryClient) DownloadStarterProject(starterProject *devfilev1.StarterProject, decryptedToken string, contextDir string, verbose bool) error {
+	return component.DownloadStarterProject(starterProject, decryptedToken, contextDir, verbose)
+}
+
+// GetDevfileRegistries gets devfile registries from preference file,
+// if registry name is specified return the specific registry, otherwise return all registries
+func (o RegistryClient) GetDevfileRegistries(registryName string) ([]Registry, error) {
+	var devfileRegistries []Registry
+
+	hasName := len(registryName) != 0
+	if o.preferenceClient.RegistryList() != nil {
+		registryList := *o.preferenceClient.RegistryList()
+		// Loop backwards here to ensure the registry display order is correct (display latest newly added registry firstly)
+		for i := len(registryList) - 1; i >= 0; i-- {
+			registry := registryList[i]
+			if hasName {
+				if registryName == registry.Name {
+					reg := Registry{
+						Name:   registry.Name,
+						URL:    registry.URL,
+						Secure: registry.Secure,
+					}
+					devfileRegistries = append(devfileRegistries, reg)
+					return devfileRegistries, nil
+				}
+			} else {
+				reg := Registry{
+					Name:   registry.Name,
+					URL:    registry.URL,
+					Secure: registry.Secure,
+				}
+				devfileRegistries = append(devfileRegistries, reg)
+			}
+		}
+	} else {
+		return nil, nil
+	}
+
+	return devfileRegistries, nil
+}
+
+// ListDevfileStacks lists all the available devfile stacks in devfile registry
+func (o RegistryClient) ListDevfileStacks(registryName string) (DevfileStackList, error) {
+	catalogDevfileList := &DevfileStackList{}
+	var err error
+
+	// TODO: consider caching registry information for better performance since it should be fairly stable over time
+	// Get devfile registries
+	catalogDevfileList.DevfileRegistries, err = o.GetDevfileRegistries(registryName)
+	if err != nil {
+		return *catalogDevfileList, err
+	}
+	if catalogDevfileList.DevfileRegistries == nil {
+		return *catalogDevfileList, nil
+	}
+
+	// first retrieve the indices for each registry, concurrently
+	devfileIndicesMutex := &sync.Mutex{}
+	retrieveRegistryIndices := util.NewConcurrentTasks(len(catalogDevfileList.DevfileRegistries))
+
+	// The 2D slice index is the priority of the registry (highest priority has highest index)
+	// and the element is the devfile slice that belongs to the registry
+	registrySlice := make([][]DevfileStack, len(catalogDevfileList.DevfileRegistries))
+	for regPriority, reg := range catalogDevfileList.DevfileRegistries {
+		// Load the devfile registry index.json
+		registry := reg                 // Needed to prevent the lambda from capturing the value
+		registryPriority := regPriority // Needed to prevent the lambda from capturing the value
+		retrieveRegistryIndices.Add(util.ConcurrentTask{ToRun: func(errChannel chan error) {
+			registryDevfiles, err := getRegistryStacks(o.preferenceClient, registry)
+			if err != nil {
+				log.Warningf("Registry %s is not set up properly with error: %v, please check the registry URL and credential (refer `odo registry update --help`)\n", registry.Name, err)
+				return
+			}
+
+			devfileIndicesMutex.Lock()
+			registrySlice[registryPriority] = registryDevfiles
+			devfileIndicesMutex.Unlock()
+		}})
+	}
+	if err := retrieveRegistryIndices.Run(); err != nil {
+		return *catalogDevfileList, err
+	}
+
+	for _, registryDevfiles := range registrySlice {
+		catalogDevfileList.Items = append(catalogDevfileList.Items, registryDevfiles...)
+	}
+
+	return *catalogDevfileList, nil
+}
+
+// convertURL converts GitHub regular URL to GitHub raw URL, do nothing if the URL is not GitHub URL
+// For example:
+// GitHub regular URL: https://github.com/elsony/devfile-registry/tree/johnmcollier-crw
+// GitHub raw URL: https://raw.githubusercontent.com/elsony/devfile-registry/johnmcollier-crw
+func convertURL(URL string) (string, error) {
+	url, err := url.Parse(URL)
+	if err != nil {
+		return "", err
+	}
+
+	if strings.Contains(url.Host, "github") && !strings.Contains(url.Host, "raw") {
+		// Convert path part of the URL
+		URLSlice := strings.Split(URL, "/")
+		if len(URLSlice) > 2 && URLSlice[len(URLSlice)-2] == "tree" {
+			// GitHub raw URL doesn't have "tree" structure in the URL, need to remove it
+			URL = strings.Replace(URL, "/tree", "", 1)
+		} else {
+			// Add "master" branch for GitHub raw URL by default if branch is not specified
+			URL = URL + "/master"
+		}
+
+		// Convert host part of the URL
+		if url.Host == "github.com" {
+			URL = strings.Replace(URL, "github.com", "raw.githubusercontent.com", 1)
+		} else {
+			URL = strings.Replace(URL, url.Host, "raw."+url.Host, 1)
+		}
+	}
+
+	return URL, nil
+}
+
+const indexPath = "/devfiles/index.json"
+
+// getRegistryStacks retrieves the registry's index devfile stack entries
+func getRegistryStacks(preferenceClient preference.Client, registry Registry) ([]DevfileStack, error) {
+	if !strings.Contains(registry.URL, "github") {
+		// OCI-based registry
+		devfileIndex, err := library.GetRegistryIndex(registry.URL, segment.GetRegistryOptions(), indexSchema.StackDevfileType)
+		if err != nil {
+			return nil, err
+		}
+		return createRegistryDevfiles(registry, devfileIndex)
+	}
+	// Github-based registry
+	URL, err := convertURL(registry.URL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to convert URL %s: %w", registry.URL, err)
+	}
+	registry.URL = URL
+	indexLink := registry.URL + indexPath
+	request := dfutil.HTTPRequestParams{
+		URL: indexLink,
+	}
+
+	secure := registryUtil.IsSecure(preferenceClient, registry.Name)
+	if secure {
+		token, e := keyring.Get(fmt.Sprintf("%s%s", dfutil.CredentialPrefix, registry.Name), registryUtil.RegistryUser)
+		if e != nil {
+			return nil, fmt.Errorf("unable to get secure registry credential from keyring: %w", e)
+		}
+		request.Token = token
+	}
+
+	jsonBytes, err := dfutil.HTTPGetRequest(request, preferenceClient.GetRegistryCacheTime())
+	if err != nil {
+		return nil, fmt.Errorf("unable to download the devfile index.json from %s: %w", indexLink, err)
+	}
+
+	var devfileIndex []indexSchema.Schema
+	err = json.Unmarshal(jsonBytes, &devfileIndex)
+	if err != nil {
+		if err := util.CleanDefaultHTTPCacheDir(); err != nil {
+			log.Warning("Error while cleaning up cache dir.")
+		}
+		// we try once again
+		jsonBytes, err := dfutil.HTTPGetRequest(request, preferenceClient.GetRegistryCacheTime())
+		if err != nil {
+			return nil, fmt.Errorf("unable to download the devfile index.json from %s: %w", indexLink, err)
+		}
+
+		err = json.Unmarshal(jsonBytes, &devfileIndex)
+		if err != nil {
+			return nil, fmt.Errorf("unable to unmarshal the devfile index.json from %s: %w", indexLink, err)
+		}
+	}
+	return createRegistryDevfiles(registry, devfileIndex)
+}
+
+func createRegistryDevfiles(registry Registry, devfileIndex []indexSchema.Schema) ([]DevfileStack, error) {
+	registryDevfiles := make([]DevfileStack, 0, len(devfileIndex))
+	for _, devfileIndexEntry := range devfileIndex {
+		stackDevfile := DevfileStack{
+			Name:        devfileIndexEntry.Name,
+			DisplayName: devfileIndexEntry.DisplayName,
+			Description: devfileIndexEntry.Description,
+			Link:        devfileIndexEntry.Links["self"],
+			Registry:    registry,
+			Language:    devfileIndexEntry.Language,
+			Tags:        devfileIndexEntry.Tags,
+			ProjectType: devfileIndexEntry.ProjectType,
+		}
+		registryDevfiles = append(registryDevfiles, stackDevfile)
+	}
+
+	return registryDevfiles, nil
+}

pkg/registry/registry_test.go

@@ -0,0 +1,198 @@
+package registry
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/redhat-developer/odo/pkg/preference"
+	"github.com/redhat-developer/odo/pkg/testingutil/filesystem"
+)
+
+func TestGetDevfileRegistries(t *testing.T) {
+	tempConfigFile, err := ioutil.TempFile("", "odoconfig")
+	if err != nil {
+		t.Fatal("Fail to create temporary config file")
+	}
+	defer os.Remove(tempConfigFile.Name())
+	defer tempConfigFile.Close()
+	_, err = tempConfigFile.Write([]byte(
+		`kind: Preference
+apiversion: odo.openshift.io/v1alpha1
+OdoSettings:
+  RegistryList:
+  - Name: DefaultDevfileRegistry
+    URL: https://registry.devfile.io
+  - Name: CheDevfileRegistry
+    URL: https://che-devfile-registry.openshift.io/`,
+	))
+	if err != nil {
+		t.Error(err)
+	}
+
+	os.Setenv(preference.GlobalConfigEnvName, tempConfigFile.Name())
+	defer os.Unsetenv(preference.GlobalConfigEnvName)
+
+	tests := []struct {
+		name         string
+		registryName string
+		want         []Registry
+	}{
+		{
+			name:         "Case 1: Test get all devfile registries",
+			registryName: "",
+			want: []Registry{
+				{
+					Name:   "CheDevfileRegistry",
+					URL:    "https://che-devfile-registry.openshift.io/",
+					Secure: false,
+				},
+				{
+					Name:   "DefaultDevfileRegistry",
+					URL:    "https://registry.devfile.io",
+					Secure: false,
+				},
+			},
+		},
+		{
+			name:         "Case 2: Test get specific devfile registry",
+			registryName: "CheDevfileRegistry",
+			want: []Registry{
+				{
+					Name:   "CheDevfileRegistry",
+					URL:    "https://che-devfile-registry.openshift.io/",
+					Secure: false,
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			prefClient, _ := preference.NewClient()
+			catClient := NewRegistryClient(filesystem.NewFakeFs(), prefClient)
+			got, err := catClient.GetDevfileRegistries(tt.registryName)
+			if err != nil {
+				t.Errorf("Error message is %v", err)
+			}
+
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Got: %v, want: %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetRegistryDevfiles(t *testing.T) {
+	// Start a local HTTP server
+	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		// Send response to be tested
+		_, err := rw.Write([]byte(
+			`
+			[
+				{
+					"name": "nodejs",
+					"displayName": "NodeJS Angular Web Application",
+					"description": "Stack for developing NodeJS Angular Web Application",
+					"tags": [
+						"NodeJS",
+						"Angular",
+						"Alpine"
+					],
+					"language": "nodejs",
+					"icon": "/images/angular.svg",
+					"globalMemoryLimit": "2686Mi",
+					"links": {
+						"self": "/devfiles/angular/devfile.yaml"
+					}
+				}
+			]
+			`,
+		))
+		if err != nil {
+			t.Error(err)
+		}
+	}))
+	// Close the server when test finishes
+	defer server.Close()
+
+	const registryName = "some registry"
+	tests := []struct {
+		name     string
+		registry Registry
+		want     []DevfileStack
+	}{
+		{
+			name:     "Test NodeJS devfile index",
+			registry: Registry{Name: registryName, URL: server.URL},
+			want: []DevfileStack{
+				{
+					Name:        "nodejs",
+					DisplayName: "NodeJS Angular Web Application",
+					Description: "Stack for developing NodeJS Angular Web Application",
+					Registry: Registry{
+						Name: registryName,
+						URL:  server.URL,
+					},
+					Link:     "/devfiles/angular/devfile.yaml",
+					Language: "nodejs",
+					Tags:     []string{"NodeJS", "Angular", "Alpine"},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			prefClient := preference.NewMockClient(ctrl)
+			got, err := getRegistryStacks(prefClient, tt.registry)
+
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Got: %v, want: %v", got, tt.want)
+				t.Logf("Error message is: %v", err)
+			}
+		})
+	}
+}
+
+func TestConvertURL(t *testing.T) {
+	tests := []struct {
+		name    string
+		URL     string
+		wantURL string
+	}{
+		{
+			name:    "Case 1: GitHub regular URL without specifying branch",
+			URL:     "https://github.com/GeekArthur/registry",
+			wantURL: "https://raw.githubusercontent.com/GeekArthur/registry/master",
+		},
+		{
+			name:    "Case 2: GitHub regular URL with master branch specified",
+			URL:     "https://github.ibm.com/Jingfu-J-Wang/registry/tree/master",
+			wantURL: "https://raw.github.ibm.com/Jingfu-J-Wang/registry/master",
+		},
+		{
+			name:    "Case 3: GitHub regular URL with non-master branch specified",
+			URL:     "https://github.com/elsony/devfile-registry/tree/johnmcollier-crw",
+			wantURL: "https://raw.githubusercontent.com/elsony/devfile-registry/johnmcollier-crw",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotURL, err := convertURL(tt.URL)
+			if err != nil {
+				t.Error(err)
+			}
+
+			if !reflect.DeepEqual(gotURL, tt.wantURL) {
+				t.Errorf("Got url: %s, want URL: %s", gotURL, tt.wantURL)
+			}
+		})
+	}
+}

pkg/registry/types.go

@@ -0,0 +1,29 @@
+package registry
+
+// Registry is the main struct of devfile registry
+type Registry struct {
+	Name   string
+	URL    string
+	Secure bool
+}
+
+// DevfileStack is the main struct for devfile catalog components
+type DevfileStack struct {
+	Name        string
+	DisplayName string
+	Description string
+	Link        string
+	Registry    Registry
+	Language    string
+	Tags        []string
+	ProjectType string
+}
+
+// DevfileStackList lists all the Devfile Stacks
+type DevfileStackList struct {
+	DevfileRegistries []Registry
+	Items             []DevfileStack
+}
+
+// TypesWithDetails is the list of project types in devfile registries, and their associated devfiles
+type TypesWithDetails map[string][]DevfileStack

pkg/registry/types_with_details.go

@@ -0,0 +1,48 @@
+package registry
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+)
+
+// GetOrderedLabels returns a list of labels for a list of project types
+func (types TypesWithDetails) GetOrderedLabels() []string {
+	sortedTypes := sortTypes(types)
+	stringTypes := []string{}
+	for _, typ := range sortedTypes {
+		detailsList := types[typ]
+		if len(detailsList) == 1 {
+			stringTypes = append(stringTypes, typ)
+		} else {
+			for _, details := range detailsList {
+				stringTypes = append(stringTypes, fmt.Sprintf("%s (%s, registry: %s)", typ, details.Name, details.Registry.Name))
+			}
+		}
+	}
+	return stringTypes
+}
+
+// GetAtOrderedPosition returns the project type at the given position,
+// when the list of project types is ordered by GetOrderedLabels
+func (types TypesWithDetails) GetAtOrderedPosition(pos int) (DevfileStack, error) {
+	sortedTypes := sortTypes(types)
+	for _, typ := range sortedTypes {
+		detailsList := types[typ]
+		if pos >= len(detailsList) {
+			pos -= len(detailsList)
+			continue
+		}
+		return detailsList[pos], nil
+	}
+	return DevfileStack{}, errors.New("index not found")
+}
+
+func sortTypes(types TypesWithDetails) []string {
+	sortedTypes := make([]string, 0, len(types))
+	for typ := range types {
+		sortedTypes = append(sortedTypes, typ)
+	}
+	sort.Strings(sortedTypes)
+	return sortedTypes
+}

pkg/registry/types_with_details_test.go

@@ -0,0 +1,222 @@
+package registry
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestTypesWithDetails_GetOrderedLabels(t *testing.T) {
+	tests := []struct {
+		name  string
+		types TypesWithDetails
+		want  []string
+	}{
+		{
+			name: "some entries",
+			types: TypesWithDetails{
+				"second devfile for lang1": []DevfileStack{
+					{
+						Name: "devfile2",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+				},
+				"first devfile for lang1": []DevfileStack{
+					{
+						Name: "devfile1",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+					{
+						Name: "devfile1",
+						Registry: Registry{
+							Name: "Registry2",
+						},
+					},
+				},
+			},
+			want: []string{
+				"first devfile for lang1 (devfile1, registry: Registry1)",
+				"first devfile for lang1 (devfile1, registry: Registry2)",
+				"second devfile for lang1",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.types.GetOrderedLabels(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("TypesWithDetails.GetOrderedLabels() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestTypesWithDetails_GetAtOrderedPosition(t *testing.T) {
+	type args struct {
+		pos int
+	}
+	tests := []struct {
+		name    string
+		types   TypesWithDetails
+		args    args
+		want    DevfileStack
+		wantErr bool
+	}{
+		{
+			name: "get a pos 0",
+			types: TypesWithDetails{
+				"second devfile for lang1": []DevfileStack{
+					{
+						Name: "devfile2",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+				},
+				"first devfile for lang1": []DevfileStack{
+					{
+						Name: "devfile1",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+					{
+						Name: "devfile1",
+						Registry: Registry{
+							Name: "Registry2",
+						},
+					},
+				},
+			},
+			args: args{
+				pos: 0,
+			},
+			want: DevfileStack{
+				Name: "devfile1",
+				Registry: Registry{
+					Name: "Registry1",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "get a pos 1",
+			types: TypesWithDetails{
+				"second devfile for lang1": []DevfileStack{
+					{
+						Name: "devfile2",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+				},
+				"first devfile for lang1": []DevfileStack{
+					{
+						Name: "devfile1",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+					{
+						Name: "devfile1",
+						Registry: Registry{
+							Name: "Registry2",
+						},
+					},
+				},
+			},
+			args: args{
+				pos: 1,
+			},
+			want: DevfileStack{
+				Name: "devfile1",
+				Registry: Registry{
+					Name: "Registry2",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "get a pos 2",
+			types: TypesWithDetails{
+				"second devfile for lang1": []DevfileStack{
+					{
+						Name: "devfile2",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+				},
+				"first devfile for lang1": []DevfileStack{
+					{
+						Name: "devfile1",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+					{
+						Name: "devfile1",
+						Registry: Registry{
+							Name: "Registry2",
+						},
+					},
+				},
+			},
+			args: args{
+				pos: 2,
+			},
+			want: DevfileStack{
+				Name: "devfile2",
+				Registry: Registry{
+					Name: "Registry1",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "get a pos 4: not found",
+			types: TypesWithDetails{
+				"second devfile for lang1": []DevfileStack{
+					{
+						Name: "devfile2",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+				},
+				"first devfile for lang1": []DevfileStack{
+					{
+						Name: "devfile1",
+						Registry: Registry{
+							Name: "Registry1",
+						},
+					},
+					{
+						Name: "devfile1",
+						Registry: Registry{
+							Name: "Registry2",
+						},
+					},
+				},
+			},
+			args: args{
+				pos: 4,
+			},
+			want:    DevfileStack{},
+			wantErr: true,
+		}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.types.GetAtOrderedPosition(tt.args.pos)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("TypesWithDetails.GetAtOrderedPosition() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("TypesWithDetails.GetAtOrderedPosition() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}