Phoenix Todos - Preloading Todos

This post is written as a set of Literate Commits. The goal of this style is to show you how this program came together from beginning to end.

Each commit in the project is represented by a section of the article. Click each section's header to see the commit on Github, or check out the repository and follow along.

Written by Pete Corey on Oct 12, 2016.

Redux Channel Actions

Connecting to our socket and joining our channel in the base of our application doesn’t feel like the React Way™.

Instead, let’s define actions and reducers to connect to the socket and join our "lists.public" channel.

We’ll start by creating a connectSocket action creator that initializes our Socket and connects it to our server. The corresponding reducer for this action will save the socket to our state:


export function connectSocket(jwt) {
  let socket = new Socket("/socket", {
    params: {
      token: jwt
    }
  });
  socket.connect();
  return { type: CONNECT_SOCKET, socket };
}

Next, let’s create a joinListsChannel action creator that joins the provided channel and dispatches addList actions for each list returned from the server:


socket
  .channel(channel)
  .join()
  .receive("ok", (lists) => {
    lists.forEach((list) => {
      dispatch(addList(list));
    });
    dispatch(joinListsChannelSuccess(channel));
  })
  .receive("error", (error) => {
    dispatch(joinListsChannelFailure(channel, error));
  });

Now we’re connecting to our Phoenix channel in a much more Redux-friendly way. Plus, we have access to our socket within our application’s state!

web/static/js/actions/index.js

+import { Socket } from "deps/phoenix/web/static/js/phoenix" + export const SIGN_UP_REQUEST = "SIGN_UP_REQUEST"; ... +export const CONNECT_SOCKET = "CONNECT_SOCKET"; + +export const JOIN_LISTS_CHANNEL_REQUEST = "JOIN_LISTS_CHANNEL_REQUEST"; +export const JOIN_LISTS_CHANNEL_SUCCESS = "JOIN_LISTS_CHANNEL_SUCCESS"; +export const JOIN_LISTS_CHANNEL_FAILURE = "JOIN_LISTS_CHANNEL_FAILURE"; + export const ADD_LIST = "ADD_LIST"; ... +export function connectSocket(jwt) { + let socket = new Socket("/socket", { + params: { + token: jwt + } + }); + socket.connect(); + return { type: CONNECT_SOCKET, socket }; +} + +export function joinListsChannelRequest(channel) { + return { type: JOIN_LISTS_CHANNEL_REQUEST, channel }; +} + +export function joinListsChannelSuccess(channel) { + return { type: JOIN_LISTS_CHANNEL_SUCCESS, channel }; +} + +export function joinListsChannelFailure(channel, error) { + return { type: JOIN_LISTS_CHANNEL_FAILURE, channel, error }; +} + export function signUp(email, password, password_confirm) { ... } + +export function joinListsChannel(channel) { + return (dispatch, getState) => { + const { socket } = getState(); + + dispatch(joinListsChannelRequest()); + + socket + .channel(channel) + .join() + .receive("ok", (lists) => { + lists.forEach((list) => { + dispatch(addList(list)); + }); + dispatch(joinListsChannelSuccess(channel)); + }) + .receive("error", (error) => { + dispatch(joinListsChannelFailure(channel, error)); + }); + } +}

web/static/js/app.js

... import { - addList + connectSocket, + joinListsChannel } from "./actions"; -import socket from "./socket"; ... render(); -store.subscribe(render); -socket.connect(); -socket.channel("lists.public", {}) - .join() - .receive("ok", (res) => { - res.forEach((list) => { - store.dispatch(addList(list)); - }); - }) - .receive("error", (res) => { - console.log("error", res); - }); +store.dispatch(connectSocket(store.getState().jwt)); +store.dispatch(joinListsChannel("lists.public"));

web/static/js/reducers/index.js

... SIGN_IN_FAILURE, + CONNECT_SOCKET, ADD_LIST, ... const initialState = { + socket: undefined, user: user ? JSON.parse(user) : user, ... }); + case CONNECT_SOCKET: + return Object.assign({}, state, { socket: action.socket }); default:

List Page

Now that our lists are being populated in the sidebar of our application, we should pull in the components, layouts, and pages necessary to render them.

We’ll grab the ListPageContainer, ListPage, ListHeader, and TodoItem React components from our original Meteor application and move them into our Phoenix project.

The main changes we’ve made to these components is renaming variables to match our Ecto models (incompleteCount to incomplete_count, and userId to user_id), and refactoring how we fetch lists.

Also, instead of using Meteor collections to fetch lists from Minimongo, we refactored our components to pull lists directly out of our application’s state:


let id = props.params.id;
let list = _.find(state.lists, list => list.id == id);

Now that we’ve added the necessary React components to our project, we can add the new ListPageContainer to our router:


<Route path="lists/:id" component={ListPageContainer}/>

Clicking on a list in our sidebar shows the (empty) list in the main panel. Success!

package.json

"brunch": "^2.0.0", + "classnames": "^2.2.5", "clean-css-brunch": ">= 1.0 < 1.8",

web/static/js/components/ListHeader.jsx

+...

web/static/js/components/ListList.jsx

... > - {list.userId + {list.user_id ? <span className="icon-lock"></span> : null} - {list.incompleteCount - ? <span className="count-list">{list.incompleteCount}</span> + {list.incomplete_count + ? <span className="count-list">{list.incomplete_count}</span> : null}

web/static/js/components/TodoItem.jsx

+...

web/static/js/containers/ListPageContainer.jsx

+import ListPage from '../pages/ListPage.jsx'; +import { connect } from "react-redux"; +import _ from "lodash"; + +const ListPageContainer = connect( + (state, props) => { + let id = props.params.id; + let list = _.find(state.lists, list => list.id == id); + return { + loading: state.loading, + list: list, + listExists: !!list, + todos: [] + } + } +)(ListPage); + +export default ListPageContainer;

web/static/js/layouts/App.jsx

... if (this.props.params.id) { - const list = Lists.findOne(this.props.params.id); - if (list.userId) { - const publicList = Lists.findOne({ userId: { $exists: false } }); + const list = _.find(this.props.lists, list => list.id == this.props.params.id); + if (list.user_id) { + const publicList = _.find(this.props.list, list => !list.user_id); this.context.router.push(`/lists/${ publicList.id }`{:.language-javascript});

web/static/js/pages/ListPage.jsx

+...

web/static/js/routes.jsx

... import NotFoundPage from './pages/NotFoundPage.jsx'; +import ListPageContainer from './containers/ListPageContainer.jsx'; ... <Router history={browserHistory}> - <Route path="/" component={AppContainer}> + <Route path="/" component={AppContainer}> + <Route path="lists/:id" component={ListPageContainer}/> <Route path="signin" component={AuthPageSignIn}/> ... <Route path="*" component={NotFoundPage}/> - </Route> + </Route> </Router>

Preloading Todos

One of the cool features of Ecto is that we can write queries that automatically load, or “preload”, related objects.

For our Todos application, we can configure our List.public query to preload all associated Todo objects:


from list in query,
where: is_nil(list.user_id),
preload: [:todos]

Now the todos field on our List will be a fully populated list of all Todo objects associated with that particular list.

To send those todos to the client, we need to tell Poison that we want the todos field included in each serialized List object:


@derive {Poison.Encoder, only: [
  ...
  :todos
]}

We’ll also need to tell Poison how to serialize our Todo documents:


@derive {Poison.Encoder, only: [
  :id,
  :text,
  :checked
]}

Now on the client, we can tell our ListPageContainer to pull our list of todos out of the list itself:


todos: _.get(list, "todos") || []

After fixing up a few minor variable name issues, our todos show up in each list page!

web/models/list.ex

... :incomplete_count, - :user_id + :user_id, + :todos ]} ... from list in query, - where: is_nil(list.user_id) + where: is_nil(list.user_id), + preload: [:todos] end

web/models/todo.ex

... + @derive {Poison.Encoder, only: [ + :id, + :text, + :checked + ]} + schema "todos" do

web/static/js/components/TodoItem.jsx

... updateText.call({ - todoId: this.props.todo._id, + todoId: this.props.todo.id, newText: value, ... onFocus() { - this.props.onEditingChange(this.props.todo._id, true); + this.props.onEditingChange(this.props.todo.id, true); } ... onBlur() { - this.props.onEditingChange(this.props.todo._id, false); + this.props.onEditingChange(this.props.todo.id, false); } ... setCheckedStatus.call({ - todoId: this.props.todo._id, + todoId: this.props.todo.id, newCheckedStatus: event.target.checked, ... deleteTodo() { - remove.call({ todoId: this.props.todo._id }, alert); + remove.call({ todoId: this.props.todo.id }, alert); }

web/static/js/containers/ListPageContainer.jsx

... listExists: !!list, - todos: [] + todos: _.get(list, "todos") || [] }

web/static/js/pages/ListPage.jsx

... todo={todo} - key={todo._id} - editing={todo._id === editingTodo} + key={todo.id} + editing={todo.id === editingTodo} onEditingChange={this.onEditingChange}

Final Thoughts

Although it was briefly glossed over in the commits, representing the WebSocket connection process and state in terms of Redux actions led to quite a bit of internal conflict.

In my mind, a highly stateful, constantly changing object like a socket or channel connection doesn’t neatly fit into the idea of “state”.

Our Redux event log could show that we successfully instantiated a socket connection, but that doesn’t mean that the socket referenced by our state is currently connected. It might have timed out, or disconnected for any other unknown reason.

Trying to capture and track this kind of transient, ephemeral thing in pure, immutable Redux state seems like a slippery and dangerous slope.

We don’t track things like network connectivity in Redux state, so why track WebSocket connectivity? That analogy isn’t exactly accurate, but I think it helps to describe some of my concerns.

Ultimately, I decided to keep the socket connection in the application state so it can be easily accessible to all components and actions.

How to Safely Store Application Links

Written by Pete Corey on Oct 10, 2016.

Sometimes your Meteor application will need to store internal application links.

Maybe you want to save the last route a user visited, or maybe you want to associate notifications with a certain route within your application.

Storing URLs

It can be tempting to store these links as full URLs in your database and render them on the client as a simple anchor tag:


<a href="{{url}}">{{link}}</a>

Don’t give into temptation! This kind of linking can be a source of danger for your users.

If a malicious user has control over the URL inserted into the database, they can link other users of your application to potentially dangerous third-party websites.

For example, an attacker could manually create a new notification and provide their own URL:


Notifications.insert({
  link: "Error dectected - please fix!",
  url: "http://www.evil-website.com"
});

Other users might see this “Error detected - please fix!” link, click it, and be redirected to http://www.evil-website.com.

Evil Website® might attempt to deceive them, extract some information from them, or even be used as a vehicle for exploiting a Cross Site Request Forgery (CSRF) vulnerability on another website.

Storing Routes

Rather than storing the entire URL in your database, only store the information necessary to recreate the URL on the client.

For example, when using Iron Router (or Flow Router), it would be sufficient to simply store the route name in your database. On the client, you could use the pathFor helper to construct the link:


<a href="{{pathFor route}}">{{link}}</a>

Similarly, in-application links can be built using the <Link> React component if your application is using React Router:


<Link to=`${route}`>{link}</Link>

Building dynamic internal links like this is a much safer alternative to using raw anchor tags. It prevents attackers from potentially linking other users of your application to malicious third-party websites.

Phoenix Todos - Public Lists

This post is written as a set of Literate Commits. The goal of this style is to show you how this program came together from beginning to end.

Each commit in the project is represented by a section of the article. Click each section's header to see the commit on Github, or check out the repository and follow along.

Written by Pete Corey on Oct 5, 2016.

List and Todo Models

We’re getting to the point where we’ll be wanting real data in our application. To use real data, we’ll need to define the schemas and models that describe the data.

Looking at our original Lists schema, we know that each list needs a name, an incompleteCount (which we’ll call incomplete_count), and an optional reference to a user.

We can use Phoenix’s phoenix.gen.model generator to create this model for us:


mix phoenix.gen.model List lists name:string \
                                 incomplete_count:integer \
                                 user_id:references:users

Running this command creates a migration to create the "users" table in our database. It also creates our PhoenixTodos.List model.

We can repeat this process for our Todos collection. Looking at the Todos schema, we know we’ll need a text field, a checked field, a reference to its parent list, and a timestamp.

Once again, we can use the phoenix.gen.model generator to create this model and migration for us:


mix phoenix.gen.model Todo todos text:string \
                                 checked:boolean \
                                 list_id:references:lists

Notice that we left the timestamp out of our generator call. Phoenix adds timestamp fields for us automatically.

Nearly all of the code generated for us is perfect. We only need to make one small tweak to our PhoenixTodos.List model. In addition to specifying that it belongs_to the PhoenixTodos.User model, we need to specify that each PhoenixTodos.List model has_many PhoenixTodos.Todo children:


has_many :todos, PhoenixTodos.Todo

Specifying this relationship on the parent List as well as the child Todo model will be very helpful down the line.

priv/repo/migrations/20160920202201_create_list.exs

+defmodule PhoenixTodos.Repo.Migrations.CreateList do + use Ecto.Migration + + def change do + create table(:lists) do + add :name, :string + add :incomplete_count, :integer + add :user_id, references(:users, on_delete: :delete_all) + + timestamps + end + create index(:lists, [:user_id]) + + end +end

priv/repo/migrations/20160920202208_create_todo.exs

+defmodule PhoenixTodos.Repo.Migrations.CreateTodo do + use Ecto.Migration + + def change do + create table(:todos) do + add :text, :string + add :checked, :boolean, default: false + add :list_id, references(:lists, on_delete: :delete_all) + + timestamps + end + create index(:todos, [:list_id]) + + end +end

test/models/list_test.exs

+defmodule PhoenixTodos.ListTest do + use PhoenixTodos.ModelCase + + alias PhoenixTodos.List + + @valid_attrs %{incomplete_count: 42, name: "some content"} + @invalid_attrs %{} + + test "changeset with valid attributes" do + changeset = List.changeset(%List{}, @valid_attrs) + assert changeset.valid? + end + + test "changeset with invalid attributes" do + changeset = List.changeset(%List{}, @invalid_attrs) + refute changeset.valid? + end +end

test/models/todo_test.exs

+defmodule PhoenixTodos.TodoTest do + use PhoenixTodos.ModelCase + + alias PhoenixTodos.Todo + + @valid_attrs %{checked: true, text: "some content"} + @invalid_attrs %{} + + test "changeset with valid attributes" do + changeset = Todo.changeset(%Todo{}, @valid_attrs) + assert changeset.valid? + end + + test "changeset with invalid attributes" do + changeset = Todo.changeset(%Todo{}, @invalid_attrs) + refute changeset.valid? + end +end

web/models/list.ex

+defmodule PhoenixTodos.List do + use PhoenixTodos.Web, :model + + schema "lists" do + field :name, :string + field :incomplete_count, :integer + belongs_to :user, PhoenixTodos.User + has_many :todos, PhoenixTodos.Todo + + timestamps + end + + @required_fields ~w(name incomplete_count) + @optional_fields ~w() + + @doc """ + Creates a changeset based on the `model` and `params`. + + If no params are provided, an invalid changeset is returned + with no validation performed. + """ + def changeset(model, params \\ :empty) do + model + |> cast(params, @required_fields, @optional_fields) + end +end

web/models/todo.ex

+defmodule PhoenixTodos.Todo do + use PhoenixTodos.Web, :model + + schema "todos" do + field :text, :string + field :checked, :boolean, default: false + belongs_to :list, PhoenixTodos.List + + timestamps + end + + @required_fields ~w(text checked) + @optional_fields ~w() + + @doc """ + Creates a changeset based on the `model` and `params`. + + If no params are provided, an invalid changeset is returned + with no validation performed. + """ + def changeset(model, params \\ :empty) do + model + |> cast(params, @required_fields, @optional_fields) + end +end

Seeding Data

Now that we’ve defined our schemas and models, we need to seed our database with data.

But before we do anything, we need to make sure that our migrations are up to date:


mix ecto.migrate

This will create our "lists" and "todos" tables our PostgreSQL database.

Now we can start writing our seeding script. We’ll model this script after the original fixtures.js file in our Meteor application.

We’ll start by creating a list of in-memory lists and todos that we’ll use to build our database objects:


[
  %{
    name: "Meteor Principles",
    items: [
      "Data on the Wire",
      "One Language",
      "Database Everywhere",
      "Latency Compensation",
      "Full Stack Reactivity",
      "Embrace the Ecosystem",
      "Simplicity Equals Productivity",
    ]
  },
  ...
]

Notice that we’re using double quote strings here instead of single quote strings, like the original Meteor appliaction. This is because single quote strings have a special meaning in Elixir.

Next, we’ll Enum.map over each object in this list. Each object represents a List, so we’ll build a List model object and insert it into our database:


list = Repo.insert!(%List{
  name: data.name,
  incomplete_count: length(data.items)
})

Each string in list.items represents a single Todo. We’ll map over this list build a new Todo model object, associating it with the List we just created using Ecto.build_assoc, and inserting it into the database:


Ecto.build_assoc(list, :todos, text: item)
|> Repo.insert!

Now we can run our seed script with the following command:


mix run priv/repo/seeds.exs

Or we can wipe our database and re-run our migrations and seed script with the following command:


mix ecto.reset

After running either of these, our database should have three lists, each with a set of associated todos.

priv/repo/seeds.exs

# mix run priv/repo/seeds.exs -# -# Inside the script, you can read and write to any of your -# repositories directly: -# -# PhoenixTodos.Repo.insert!(%PhoenixTodos.SomeModel{}) -# -# We recommend using the bang functions (`insert!`, `update!` -# and so on) as they will fail if something goes wrong. + +alias PhoenixTodos.{Repo, List} + +[ + %{ + name: "Meteor Principles", + items: [ + "Data on the Wire", + "One Language", + "Database Everywhere", + "Latency Compensation", + "Full Stack Reactivity", + "Embrace the Ecosystem", + "Simplicity Equals Productivity", + ] + }, + %{ + name: "Languages", + items: [ + "Lisp", + "C", + "C++", + "Python", + "Ruby", + "JavaScript", + "Scala", + "Erlang", + "6502 Assembly", + ] + }, + %{ + name: "Favorite Scientists", + items: [ + "Ada Lovelace", + "Grace Hopper", + "Marie Curie", + "Carl Friedrich Gauss", + "Nikola Tesla", + "Claude Shannon", + ] + } +] +|> Enum.map(fn data -> + list = Repo.insert!(%List{ + name: data.name, + incomplete_count: length(data.items) + }) + Enum.map(data.items, fn item -> + Ecto.build_assoc(list, :todos, text: item) + |> Repo.insert! + end) +end)

Public Lists

Now that our database is populated with Lists and Todos, we’re in a position where we can start passing this data down the the client.

To keep things as similar to our original Meteor application as possible, we’ll be doing all of our commuication via WebSockets. Specifically, we’ll be using Phoenix Channels.

We’ll start by creating a "lists.public" channel. This channel will emulate the "lists.public" publication in our Meteor application:


channel "lists.public", PhoenixTodos.ListChannel

When a client joins this channel, we’ll send them all public lists:


lists = List |> List.public |> Repo.all
{:ok, lists, socket}

Where public lists are lists without an associated User:


def public(query) do
  from list in query,
  where: is_nil(list.user_id)
end

In order to send these lists down the wire, we need to use Poison to tell Phoenix how to serialize our List objects into JSON:


@derive {Poison.Encoder, only: [
  :id,
  :name,
  :incomplete_count,
  :user_id
]}

Now our client can connect to our server and join the "lists.public" channel:


socket.connect();
socket.channel("lists.public", {})
  .join()

For each of the lists we receive back, well fire an ADD_LIST Redux action. The resolver for this action simply pushes the List object onto our application’s lists array:


return Object.assign({}, state, {
  lists: [...state.lists, action.list]
});

And with that (and a few minor bug fixes), our application is now showing lists pulled from the server!

test/models/list_test.exs

... alias PhoenixTodos.List + alias PhoenixTodos.User + alias PhoenixTodos.Repo ... end + + test "public" do + user = User.changeset(%User{}, %{ + email: "user@example.com", + password: "password" + }) |> Repo.insert! + public = Repo.insert!(%List{ + name: "public", + incomplete_count: 1 + }) + Repo.insert!(%List{ + name: "private", + incomplete_count: 1, + user_id: user.id + }) + + lists = List |> List.public |> Repo.all + + assert lists == [public] + end end

web/channels/list_channel.ex

+defmodule PhoenixTodos.ListChannel do + use Phoenix.Channel + alias PhoenixTodos.{Repo, List} + + def join("lists.public", _message, socket) do + lists = List |> List.public |> Repo.all + {:ok, lists, socket} + end + +end

web/channels/user_socket.ex

... # channel "rooms:*", PhoenixTodos.RoomChannel + channel "lists.public", PhoenixTodos.ListChannel

web/models/list.ex

... + @derive {Poison.Encoder, only: [ + :id, + :name, + :incomplete_count, + :user_id + ]} + schema "lists" do ... end + + def public(query) do + from list in query, + where: is_nil(list.user_id) + end + end

web/static/js/actions/index.js

... +export const ADD_LIST = "ADD_LIST"; + export function signUpRequest() { ... +export function addList(list) { + return { type: ADD_LIST, list }; +} + export function signUp(email, password, password_confirm) {

web/static/js/app.js

... import thunkMiddleware from "redux-thunk"; +import { + addList +} from "./actions"; +import socket from "./socket"; ... store.subscribe(render); + +socket.connect(); +socket.channel("lists.public", {}) + .join() + .receive("ok", (res) => { + res.forEach((list) => { + store.dispatch(addList(list)); + }); + }) + .receive("error", (res) => { + console.log("error", res); + });

web/static/js/components/ListList.jsx

... <Link - to={`/lists/${ list._id }`{:.language-javascript}} - key={list._id} + to={`/lists/${ list.id }`{:.language-javascript}} + key={list.id} title={list.name}

web/static/js/layouts/App.jsx

... // redirect / to a list once lists are ready - if (!loading && !children) { - const list = Lists.findOne(); - this.context.router.replace(`/lists/${ list._id }`{:.language-javascript}); + if (!loading && !children && this.props.lists.length) { + const list = this.props.lists[0]; + this.context.router.replace(`/lists/${ list.id }`{:.language-javascript}); } ... const publicList = Lists.findOne({ userId: { $exists: false } }); - this.context.router.push(`/lists/${ publicList._id }`{:.language-javascript}); + this.context.router.push(`/lists/${ publicList.id }`{:.language-javascript}); }

web/static/js/reducers/index.js

... SIGN_IN_FAILURE, + ADD_LIST, } from "../actions"; ... }); + case ADD_LIST: + return Object.assign({}, state, { + lists: [...state.lists, action.list] + }); default:

web/static/js/socket.js

... -// Now that you are connected, you can join channels with a topic: -let channel = socket.channel("topic:subtopic", {}) -channel.join() - .receive("ok", resp => { console.log("Joined successfully", resp) }) - .receive("error", resp => { console.log("Unable to join", resp) }) - export default socket